From b7425f1a7738b413f9869f54e35b1537cb583b3d Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 28 Mar 2023 14:55:23 +0100 Subject: [PATCH 001/288] initial commit --- indica/models/magnetic_recon.py | 87 +++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 indica/models/magnetic_recon.py diff --git a/indica/models/magnetic_recon.py b/indica/models/magnetic_recon.py new file mode 100644 index 00000000..1cc3667a --- /dev/null +++ b/indica/models/magnetic_recon.py @@ -0,0 +1,87 @@ +import xarray as xr +import matplotlib.pyplot as plt + +from indica.models.abstractdiagnostic import DiagnosticModel +from indica.models.plasma import example_run as example_plasma +from indica.readers.available_quantities import AVAILABLE_QUANTITIES + + +class MagneticRecon(DiagnosticModel): + """ + Object representing observations from a magnetic reconstruction + """ + + def __init__( + self, + name: str, + instrument_method="get_equilibrium", + ): + self.name = name + self.instrument_method = instrument_method + self.quantities = AVAILABLE_QUANTITIES[self.instrument_method] + + def _build_bckc_dictionary(self): + self.bckc = {} + + for quant in self.quantities: + datatype = self.quantities[quant] + if quant == "wp": + self.bckc[quant] = self.wp + error = xr.full_like(self.bckc[quant], 0.0) + stdev = xr.full_like(self.bckc[quant], 0.0) + self.bckc[quant].attrs = { + "datatype": datatype, + "error": error, + "stdev": stdev, + "provenance": str(self), + } + else: + # print(f"{quant} not available in model for {self.instrument_method}") + continue + + def __call__( + self, + t = None, + **kwargs, + ): + """ + + Returns + ------- + bckc values + """ + if self.plasma is None: + raise ValueError("plasma object is needed") + + if t is None: + t = self.plasma.time_to_calculate + + self.wp = self.plasma.wp.sel(t=t) + self._build_bckc_dictionary() + return self.bckc + + +def example_run( + diagnostic_name: str = "efit", + plasma=None, + plot=False, + t=None, +): + if plasma is None: + plasma = example_plasma() + + model = MagneticRecon( + diagnostic_name, + ) + model.set_plasma(plasma) + bckc = model() + + if plot: + bckc["wp"].plot() + + return plasma, model, bckc + + +if __name__ == "__main__": + example_run(plot=True) + plt.show(block=True) From aae4c4726ecd1c0d9cd28a8393a7e78079f98f9e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 28 Mar 2023 14:55:46 +0100 Subject: [PATCH 002/288] magnetic recon tests --- tests/unit/models/test_diagnostic_models.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/models/test_diagnostic_models.py b/tests/unit/models/test_diagnostic_models.py index 5055a93a..00323ab8 100644 --- a/tests/unit/models/test_diagnostic_models.py +++ b/tests/unit/models/test_diagnostic_models.py @@ -11,6 +11,8 @@ from indica.models.interferometry import example_run as interf from indica.models.plasma import example_run as example_plasma from indica.models.thomson_scattering import example_run as ts +from indica.models.magnetic_recon import example_run as mag_recon + EXAMPLES: Dict[str, Callable] = { "bolometer_camera": bolo, @@ -19,6 +21,7 @@ "helike_spectroscopy": helike, "thomson_scattering": ts, "charge_exchange": cxrs, + "magnetic_reconstruction":mag_recon } PLASMA = example_plasma(pulse=None, tstart=0, tend=0.1, dt=0.02) NT = np.size(PLASMA.t) @@ -124,3 +127,15 @@ def test_helike_timepoint_pass(): def test_helike_interpolation(): _test_time_interpolation("helike_spectroscopy") + + +def test_magrecon_timepoint_fail(): + _test_timepoint_fail("magnetic_reconstruction") + + +def test_magrecon_timepoint_pass(): + _test_timepoint_pass("magnetic_reconstruction") + + +def test_magrecon_interpolation(): + _test_time_interpolation("magnetic_reconstruction") \ No newline at end of file From 3100262d7702c7917923a510d5f13c90c7b25535 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 30 Mar 2023 10:06:18 +0100 Subject: [PATCH 003/288] use error attribute if it is given --- indica/bayesmodels.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index bba9d892..6a4c2ef4 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -59,7 +59,17 @@ def __init__( raise ValueError(f"{missing_data} not found in data given") def _build_bckc(self, params: dict, **kwargs): - # TODO: consider how to handle if models have overlapping kwargs + """ + TODO: consider how to handle if models have overlapping kwargs + Parameters + ---------- + params - dictionary which is updated by optimiser + kwargs - passed to model i.e. settings + + Returns + ------- + bckc of results + """ # Params is a dictionary which is updated by optimiser, # kwargs is constant i.e. settings for models self.bckc: dict = {} @@ -73,22 +83,18 @@ def _build_bckc(self, params: dict, **kwargs): def _ln_likelihood(self): ln_likelihood = 0 for key in self.quant_to_optimise: - # TODO: What to use as error? Assume percentage error if none given... # Float128 is used since rounding of small numbers causes # problems when initial results are bad fits model_data = self.bckc[key].values.astype("float128") - exp_data = ( - self.data[key] - .sel(t=self.plasma.time_to_calculate) - .values.astype("float128") - ) - _ln_likelihood = np.log( - gaussian( - model_data, - exp_data, - exp_data * 0.10 - ) - ) + exp_data = self.data[key].sel(t=self.plasma.time_to_calculate).values.astype("float128") + if hasattr(self.data[key], "error"): # Assume percentage error if none given. + if (self.data[key].error != 0).any(): # TODO: Some models have an error of 0 given + exp_error = self.data[key].error.values + else: + exp_error = exp_data * 0.10 + else: + exp_error = exp_data * 0.10 + _ln_likelihood = np.log(gaussian(model_data, exp_data, exp_error)) ln_likelihood += np.nanmean(_ln_likelihood) return ln_likelihood From ba574ecc804e2d2a1518fbf1b8c07a58965e5ae4 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 30 Mar 2023 10:06:56 +0100 Subject: [PATCH 004/288] Added poisson noise and adjusted calibration factor --- indica/models/helike_spectroscopy.py | 132 ++++++++++++++------------- 1 file changed, 68 insertions(+), 64 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index e5a1cd4f..311aeea7 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -36,17 +36,17 @@ class Helike_spectroscopy(DiagnosticModel): """ def __init__( - self, - name: str, - instrument_method="get_helike_spectroscopy", - etendue: float = 1.0, - calibration: float = 1.0e-27, - marchuk: bool = True, - full_run: bool = False, - element: str = "ar", - window_len: int = 1030, - window_lim: list = [0.394, 0.401], - window_masks: list = [], + self, + name: str, + instrument_method="get_helike_spectroscopy", + etendue: float = 1.0, + calibration: float = 5.0e-18, + marchuk: bool = True, + full_run: bool = False, + element: str = "ar", + window_len: int = 1030, + window_lim: list = [0.394, 0.401], + window_masks: list = [], ): """ Read all atomic data and initialise objects @@ -159,8 +159,8 @@ def _set_adas_pecs(self): for line in pec: pec[line]["emiss_coeff"] = ( pec[line]["emiss_coeff"] - .sel(electron_density=4.0e19, method="nearest") - .drop_vars("electron_density") + .sel(electron_density=4.0e19, method="nearest") + .drop_vars("electron_density") ) self.pec = pec @@ -231,27 +231,27 @@ def _calculate_line_emission(self, line_labels=["w", "k"]): ) for line_name in line_labels: _line_emission = ( - _pecs.isel( - line_name=(_pecs.wavelength < self.line_ranges[line_name].stop) - & (_pecs.wavelength > self.line_ranges[line_name].start) - ) - * mult + _pecs.isel( + line_name=(_pecs.wavelength < self.line_ranges[line_name].stop) + & (_pecs.wavelength > self.line_ranges[line_name].start) + ) + * mult ) _line_emission = _line_emission.sum(["type", "line_name"]) # TODO: convert PEC wavelengths to nm as per convention at TE! ev_wavelength = ph.nm_eV_conversion(nm=self.line_ranges[line_name].start) line_emission[line_name] = ( - xr.where(_line_emission >= 0, _line_emission, 0) * ev_wavelength - ) * self.calibration + xr.where(_line_emission >= 0, _line_emission, 0) * ev_wavelength + ) * self.calibration if "k" in line_emission.keys() and "w" in line_emission.keys(): line_emission["kw"] = line_emission["k"] * line_emission["w"] if "n3" in line_emission.keys() and "w" in line_emission.keys(): line_emission["n3w"] = line_emission["n345"] * line_emission["w"] if ( - "n3" in line_emission.keys() - and "n345" in line_emission.keys() - and "w" in line_emission.keys() + "n3" in line_emission.keys() + and "n345" in line_emission.keys() + and "w" in line_emission.keys() ): line_emission["tot"] = line_emission["n345"] + line_emission["w"] line_emission["n3tot"] = line_emission["n345"] * line_emission["w"] @@ -280,6 +280,8 @@ def _calculate_los_integral(self, calc_rho=False): t=self.spectra.t, calc_rho=calc_rho, ) + # TODO: LOS integral removes NaNs so manually add them back (find better solution) + self.measured_spectra[self.measured_spectra == 0] = np.nan def _calculate_temperatures(self): x1 = self.los_transform.x1 @@ -317,10 +319,10 @@ def _calculate_temperatures(self): ) def _moment_analysis( - self, - line: str, - profile_1d: DataArray = None, - half_los: bool = True, + self, + line: str, + profile_1d: DataArray = None, + half_los: bool = True, ): """ Perform moment analysis using a specific line emission as distribution function @@ -453,7 +455,7 @@ def _moment_analysis( return result, pos, err_in, err_out def _calculate_intensity( - self, + self, ): """ Returns DataArrays of emission type with co-ordinates of line label and @@ -467,8 +469,8 @@ def _calculate_intensity( _pecs_ds = _pecs.to_dataset("type") temp = [ _pecs_ds[type] - .dropna("line_name", how="all") - .interp(electron_temperature=self.Te, assume_sorted=True) + .dropna("line_name", how="all") + .interp(electron_temperature=self.Te, assume_sorted=True) for type in _pecs_ds.data_vars.keys() ] _intensity = xr.merge(temp).to_array("type") @@ -476,7 +478,7 @@ def _calculate_intensity( return intensity def _make_spectra( - self, + self, ): """ TODO: Doppler Shift / Add convolution of broadening @@ -525,29 +527,29 @@ def _build_bckc_dictionary(self): datatype = self.quantities[quant] if datatype[0] == ("intensity"): line = str(quant.split("_")[1]) - quantity = f"int_{line}" - self.bckc[quantity] = self.measured_intensity[line] - self.bckc[quantity].attrs["emiss"] = self.line_emission[line] + quant = f"int_{line}" + self.bckc[quant] = self.measured_intensity[line] + self.bckc[quant].attrs["emiss"] = self.line_emission[line] elif datatype == ("temperature", "electrons"): line = str(quant.split("_")[1]) - quantity = f"te_{line}" - self.bckc[quantity] = self.measured_Te[line] - self.bckc[quantity].attrs["emiss"] = self.line_emission[line] + quant = f"te_{line}" + self.bckc[quant] = self.measured_Te[line] + self.bckc[quant].attrs["emiss"] = self.line_emission[line] elif datatype == ("temperature", "ions"): line = str(quant.split("_")[1]) - quantity = f"ti_{line}" - self.bckc[quantity] = self.measured_Ti[line] - self.bckc[quantity].attrs["emiss"] = self.line_emission[line] + quant = f"ti_{line}" + self.bckc[quant] = self.measured_Ti[line] + self.bckc[quant].attrs["emiss"] = self.line_emission[line] elif datatype == ("spectra", "passive"): - quantity = quant self.bckc["spectra"] = self.measured_spectra + self.bckc["spectra"].attrs["error"] = np.sqrt(self.measured_spectra) # poisson noise else: print(f"{quant} not available in model for {self.instrument_method}") continue - self.bckc[quantity].attrs["datatype"] = datatype + self.bckc[quant].attrs["datatype"] = datatype if quant != "spectra" and hasattr(self, "spectra"): - self.bckc[quantity].attrs["pos"] = { + self.bckc[quant].attrs["pos"] = { "value": self.pos[line], "err_in": self.err_in[line], "err_out": self.err_out[line], @@ -560,19 +562,19 @@ def _build_bckc_dictionary(self): self.bckc["int_n3/int_tot"] = self.bckc["int_n3"] / self.bckc["int_tot"] def __call__( - self, - Te: DataArray = None, - Ti: DataArray = None, - Ne: DataArray = None, - Nimp: DataArray = None, - Fz: dict = None, - Nh: DataArray = None, - t: LabeledArray = None, - calc_spectra=True, - calc_rho: bool = False, - minimum_lines: bool = False, - moment_analysis: bool = False, - **kwargs, + self, + Te: DataArray = None, + Ti: DataArray = None, + Ne: DataArray = None, + Nimp: DataArray = None, + Fz: dict = None, + Nh: DataArray = None, + t: LabeledArray = None, + calc_spectra=True, + calc_rho: bool = False, + minimum_lines: bool = False, + moment_analysis: bool = False, + **kwargs, ): """ Calculate diagnostic measured values @@ -622,12 +624,12 @@ def __call__( Nimp = self.plasma.impurity_density.sel(t=t) else: if ( - Ne is None - or Te is None - or Nh is None - or Fz is None - or Ti is None - or Nimp is None + Ne is None + or Te is None + or Nh is None + or Fz is None + or Ti is None + or Nimp is None ): raise ValueError("Give inputs or assign plasma class!") @@ -649,7 +651,7 @@ def __call__( if minimum_lines: line_labels = [ "w", - "k", + "k" ] names = ["int_w", "int_k", "te_kw", "ti_w"] else: @@ -662,7 +664,7 @@ def __call__( "te_n3w", "te_kw", "ti_z", - "ti_w", + "ti_w" ] quant = dict(quant, **{x: self.quantities[x] for x in names}) @@ -700,6 +702,8 @@ def doppler_broaden(x, integral, center, ion_mass, ion_temp): def gaussian(x, integral, center, sigma): return (integral / (sigma * np.sqrt(2 * np.pi)) * np.exp(-((x - center) ** 2) / (2 * sigma ** 2))) + + # fmt: on From d5fec79bbbd25a34805422dcd82c2f78e038596f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 30 Mar 2023 10:41:05 +0100 Subject: [PATCH 005/288] changed name magnetic_recon -> equilibrium_reconstruction --- ...ic_recon.py => equilibrium_reconstruction.py} | 6 +++--- tests/unit/models/test_diagnostic_models.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) rename indica/models/{magnetic_recon.py => equilibrium_reconstruction.py} (94%) diff --git a/indica/models/magnetic_recon.py b/indica/models/equilibrium_reconstruction.py similarity index 94% rename from indica/models/magnetic_recon.py rename to indica/models/equilibrium_reconstruction.py index 1cc3667a..95dff261 100644 --- a/indica/models/magnetic_recon.py +++ b/indica/models/equilibrium_reconstruction.py @@ -6,7 +6,7 @@ from indica.readers.available_quantities import AVAILABLE_QUANTITIES -class MagneticRecon(DiagnosticModel): +class EquilibriumReconstruction(DiagnosticModel): """ Object representing observations from a magnetic reconstruction """ @@ -70,8 +70,8 @@ def example_run( if plasma is None: plasma = example_plasma() - model = MagneticRecon( - diagnostic_name, + model = EquilibriumReconstruction( + diagnostic_name ) model.set_plasma(plasma) bckc = model() diff --git a/tests/unit/models/test_diagnostic_models.py b/tests/unit/models/test_diagnostic_models.py index 00323ab8..2e01a538 100644 --- a/tests/unit/models/test_diagnostic_models.py +++ b/tests/unit/models/test_diagnostic_models.py @@ -11,7 +11,7 @@ from indica.models.interferometry import example_run as interf from indica.models.plasma import example_run as example_plasma from indica.models.thomson_scattering import example_run as ts -from indica.models.magnetic_recon import example_run as mag_recon +from indica.models.equilibrium_reconstruction import example_run as equil_recon EXAMPLES: Dict[str, Callable] = { @@ -21,7 +21,7 @@ "helike_spectroscopy": helike, "thomson_scattering": ts, "charge_exchange": cxrs, - "magnetic_reconstruction":mag_recon + "equilibrium_reconstruction": equil_recon, } PLASMA = example_plasma(pulse=None, tstart=0, tend=0.1, dt=0.02) NT = np.size(PLASMA.t) @@ -129,13 +129,13 @@ def test_helike_interpolation(): _test_time_interpolation("helike_spectroscopy") -def test_magrecon_timepoint_fail(): - _test_timepoint_fail("magnetic_reconstruction") +def test_equil_recon_timepoint_fail(): + _test_timepoint_fail("equilibrium_reconstruction") -def test_magrecon_timepoint_pass(): - _test_timepoint_pass("magnetic_reconstruction") +def test_equil_recon_timepoint_pass(): + _test_timepoint_pass("equilibrium_reconstruction") -def test_magrecon_interpolation(): - _test_time_interpolation("magnetic_reconstruction") \ No newline at end of file +def test_equil_recon_interpolation(): + _test_time_interpolation("equilibrium_reconstruction") \ No newline at end of file From 56c769a83929eb76ff0f92e0a7b6340e276685fa Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 30 Mar 2023 15:31:35 +0100 Subject: [PATCH 006/288] diagnostic data with channel dim are treated as individual measurements --- indica/bayesmodels.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 6a4c2ef4..c99263c6 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -55,12 +55,12 @@ def __init__( diag_model.plasma = self.plasma missing_data = list(set(quant_to_optimise).difference(data.keys())) - if missing_data: # list of keys in quant_to_optimise but not data + if missing_data: # gives list of keys in quant_to_optimise but not data raise ValueError(f"{missing_data} not found in data given") def _build_bckc(self, params: dict, **kwargs): """ - TODO: consider how to handle if models have overlapping kwargs + TODO: consider how to handle when models have overlapping kwargs Parameters ---------- params - dictionary which is updated by optimiser @@ -83,19 +83,20 @@ def _build_bckc(self, params: dict, **kwargs): def _ln_likelihood(self): ln_likelihood = 0 for key in self.quant_to_optimise: - # Float128 is used since rounding of small numbers causes - # problems when initial results are bad fits - model_data = self.bckc[key].values.astype("float128") - exp_data = self.data[key].sel(t=self.plasma.time_to_calculate).values.astype("float128") - if hasattr(self.data[key], "error"): # Assume percentage error if none given. + # Float128 since rounding of small numbers causes problems when initial results are bad fits + model_data = self.bckc[key].astype("float128") + exp_data = self.data[key].sel(t=self.plasma.time_to_calculate).astype("float128") + + exp_error = exp_data * 0.10 # Assume percentage error if none given. + if hasattr(self.data[key], "error"): if (self.data[key].error != 0).any(): # TODO: Some models have an error of 0 given - exp_error = self.data[key].error.values - else: - exp_error = exp_data * 0.10 - else: - exp_error = exp_data * 0.10 + exp_error = self.data[key].error + _ln_likelihood = np.log(gaussian(model_data, exp_data, exp_error)) - ln_likelihood += np.nanmean(_ln_likelihood) + # Treat channel as key dim which isn't averaged like other dims + if "channel" in _ln_likelihood.dims: + _ln_likelihood = _ln_likelihood.sum(dim="channel", skipna=True) + ln_likelihood += _ln_likelihood.mean(skipna=True).values return ln_likelihood def _ln_prior(self, parameters: dict): From 91d7e66d08565b2f1709090f62973b6e95ac6b44 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 31 Mar 2023 13:22:59 +0100 Subject: [PATCH 007/288] added function of example LOS --- indica/models/helike_spectroscopy.py | 39 ++++++++++--------- .../models/test_diagnostic_models.py | 0 .../models/test_plasma.py | 0 3 files changed, 21 insertions(+), 18 deletions(-) rename tests/{unit => integration}/models/test_diagnostic_models.py (100%) rename tests/{unit => integration}/models/test_plasma.py (100%) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index e5a1cd4f..175ee401 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -742,16 +742,7 @@ def select_transition(adf15_data, transition: str, wavelength: float): return pec -def example_run(pulse: int = 9229, plasma=None, plot=False, **kwargs): - # TODO: LOS sometimes crossing bad EFIT reconstruction - if plasma is None: - plasma = example_plasma( - pulse=pulse, impurities=("ar",), impurity_concentration=(0.001,), n_rad=10 - ) - plasma.time_to_calculate = plasma.t[5] - # Create new diagnostic - diagnostic_name = "xrcs" - nchannels = 3 +def helike_LOS_example(nchannels=3): los_end = np.full((nchannels, 3), 0.0) los_end[:, 0] = 0.17 los_end[:, 1] = 0.0 @@ -761,16 +752,28 @@ def example_run(pulse: int = 9229, plasma=None, plot=False, **kwargs): direction = los_end - los_start los_transform = LineOfSightTransform( - origin[:, 0], - origin[:, 1], - origin[:, 2], - direction[:, 0], - direction[:, 1], - direction[:, 2], - name=diagnostic_name, - machine_dimensions=plasma.machine_dimensions, + origin[0:nchannels, 0], + origin[0:nchannels, 1], + origin[0:nchannels, 2], + direction[0:nchannels, 0], + direction[0:nchannels, 1], + direction[0:nchannels, 2], + name="diagnostic_name", + machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), passes=1, ) + return los_transform + +def example_run(pulse: int = 9229, plasma=None, plot=False, **kwargs): + # TODO: LOS sometimes crossing bad EFIT reconstruction + if plasma is None: + plasma = example_plasma( + pulse=pulse, impurities=("ar",), impurity_concentration=(0.001,), n_rad=10 + ) + plasma.time_to_calculate = plasma.t[5] + # Create new diagnostic + diagnostic_name = "xrcs" + los_transform = helike_LOS_example(3) los_transform.set_equilibrium(plasma.equilibrium) model = Helike_spectroscopy( diagnostic_name, diff --git a/tests/unit/models/test_diagnostic_models.py b/tests/integration/models/test_diagnostic_models.py similarity index 100% rename from tests/unit/models/test_diagnostic_models.py rename to tests/integration/models/test_diagnostic_models.py diff --git a/tests/unit/models/test_plasma.py b/tests/integration/models/test_plasma.py similarity index 100% rename from tests/unit/models/test_plasma.py rename to tests/integration/models/test_plasma.py From 37cd733a116258f3e7c8115b274e3122ece9ce9c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 31 Mar 2023 13:53:56 +0100 Subject: [PATCH 008/288] refactoring --- .../models/test_helike_spectroscopy.py | 83 ++++++------------- 1 file changed, 24 insertions(+), 59 deletions(-) diff --git a/tests/integration/models/test_helike_spectroscopy.py b/tests/integration/models/test_helike_spectroscopy.py index c0ccb061..53ece85f 100644 --- a/tests/integration/models/test_helike_spectroscopy.py +++ b/tests/integration/models/test_helike_spectroscopy.py @@ -1,75 +1,40 @@ -import numpy as np - -from indica.converters.line_of_sight import LineOfSightTransform import indica.models.helike_spectroscopy as helike +from indica.models.helike_spectroscopy import helike_LOS_example from indica.models.plasma import example_run as example_plasma +import pytest -# Setup LOS -def helike_LOS_example(nchannels=3): - los_end = np.full((nchannels, 3), 0.0) - los_end[:, 0] = 0.17 - los_end[:, 1] = 0.0 - los_end[:, 2] = np.linspace(0.43, -0.43, nchannels) - los_start = np.array([[0.8, 0, 0]] * los_end.shape[0]) - origin = los_start - direction = los_end - los_start - - los_transform = LineOfSightTransform( - origin[0:nchannels, 0], - origin[0:nchannels, 1], - origin[0:nchannels, 2], - direction[0:nchannels, 0], - direction[0:nchannels, 1], - direction[0:nchannels, 2], - name="diagnostic_name", - machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), - passes=1, - ) - return los_transform - +plasma = example_plasma(tstart=0, tend=0.1, dt=0.02) class TestHelike: def setup_class(self): - self.plasma = example_plasma(pulse=9229) - self.single_time_point = self.plasma.time_to_calculate[1] - self.multiple_time_point = self.plasma.time_to_calculate + self.single_time_point = plasma.time_to_calculate[1] + self.multiple_time_point = plasma.time_to_calculate self.multiple_channel_los_transform = helike_LOS_example(nchannels=3) self.single_channel_los_transform = helike_LOS_example(nchannels=1) - self.single_channel_los_transform.set_equilibrium(self.plasma.equilibrium) - self.multiple_channel_los_transform.set_equilibrium(self.plasma.equilibrium) + self.single_channel_los_transform.set_equilibrium(plasma.equilibrium) + self.multiple_channel_los_transform.set_equilibrium(plasma.equilibrium) - def test_plasma_time_to_calculate_is_longer_than_one(self): - assert self.plasma.time_to_calculate.__len__() > 1 + def setup_method(self): + self.model = helike.Helike_spectroscopy("diagnostic_name") + self.model.set_plasma(plasma) + + def test_helike_moment_runs_with_multiple_LOS(self, ): + self.model.set_los_transform(self.multiple_channel_los_transform) + bckc = self.model(calc_spectra=False, moment_analysis=True) + assert bckc - def test_helike_runs_with_example_plasma_and_multiple_LOS(self): - model = helike.Helike_spectroscopy( - "diagnostic_name", - ) - self.plasma.time_to_calculate = self.multiple_time_point - model.set_plasma(self.plasma) - model.set_los_transform(self.multiple_channel_los_transform) - bckc = model(calc_spectra=False, moment_analysis=True) + def test_helike_moment_runs_with_single_LOS(self, ): + self.model.set_los_transform(self.single_channel_los_transform) + bckc = self.model(calc_spectra=False, moment_analysis=True) assert bckc - def test_helike_runs_with_example_plasma_and_single_LOS_and_multiple_time_point( - self, - ): - model = helike.Helike_spectroscopy( - "diagnostic_name", - ) - model.set_plasma(self.plasma) - self.plasma.time_to_calculate = self.multiple_time_point - model.set_los_transform(self.single_channel_los_transform) - bckc = model(calc_spectra=False, moment_analysis=True) + def test_helike_spectra_with_multiple_LOS(self, ): + self.model.set_los_transform(self.multiple_channel_los_transform) + bckc = self.model(calc_spectra=True, moment_analysis=False) assert bckc - def test_helike_runs_with_example_plasma_and_single_LOS_and_single_time_point(self): - model = helike.Helike_spectroscopy( - "diagnostic_name", - ) - model.set_plasma(self.plasma) - self.plasma.time_to_calculate = self.single_time_point - model.set_los_transform(self.single_channel_los_transform) - bckc = model(calc_spectra=False, moment_analysis=True) + def test_helike_spectra_with_single_LOS(self, ): + self.model.set_los_transform(self.single_channel_los_transform) + bckc = self.model(calc_spectra=True, moment_analysis=False) assert bckc From 6dbebc64c225071fa644ed4c3807b61967628b15 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 31 Mar 2023 13:54:09 +0100 Subject: [PATCH 009/288] refactoring --- tests/integration/test_bayesmodels.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/integration/test_bayesmodels.py b/tests/integration/test_bayesmodels.py index 34aaa020..122e43eb 100644 --- a/tests/integration/test_bayesmodels.py +++ b/tests/integration/test_bayesmodels.py @@ -10,7 +10,7 @@ class TestBayesModels: def setup_class(self): - self.plasma = example_run(pulse=9229) + self.plasma = example_run() self.plasma.time_to_calculate = self.plasma.t[1] self.los_transform = helike_LOS_example(nchannels=1) self.los_transform.set_equilibrium(self.plasma.equilibrium) @@ -24,9 +24,7 @@ def test_simple_run_bayesmodels_with_xrcs(self): priors = { "Te_prof.y0": get_uniform(2e3, 5e3), - # "Te_prof_peaking": get_uniform(1, 5), "Ti_prof.y0": get_uniform(2e3, 8e3), - # "Ti_prof_peaking": get_uniform(1, 5), } bckc = {} @@ -47,9 +45,7 @@ def test_simple_run_bayesmodels_with_xrcs(self): # Setup Optimiser param_names = [ "Te_prof.y0", - # "Te_prof.peaking", "Ti_prof.y0", - # "Ti_prof.peaking", ] ndim = param_names.__len__() From 57c6a665e6d9b3994ca1d80841519979a5010afa Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 31 Mar 2023 13:54:54 +0100 Subject: [PATCH 010/288] refactoring --- tests/integration/test_bayesmodels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_bayesmodels.py b/tests/integration/test_bayesmodels.py index 122e43eb..72cb7295 100644 --- a/tests/integration/test_bayesmodels.py +++ b/tests/integration/test_bayesmodels.py @@ -1,6 +1,6 @@ import emcee import flatdict -from tests.integration.models.test_helike_spectroscopy import helike_LOS_example +from indica.models.helike_spectroscopy import helike_LOS_example from indica.bayesmodels import BayesModels from indica.bayesmodels import get_uniform From ac904b60b495139ebac03f8ebb11f8b395c70850 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 31 Mar 2023 13:57:16 +0100 Subject: [PATCH 011/288] initial commit --- tests/unit/models/__init__.py | 0 tests/unit/models/test_plasma.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/unit/models/__init__.py create mode 100644 tests/unit/models/test_plasma.py diff --git a/tests/unit/models/__init__.py b/tests/unit/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/models/test_plasma.py b/tests/unit/models/test_plasma.py new file mode 100644 index 00000000..e69de29b From 4fa0bb1e4948ab3498d8c76fc3196f363aa52495 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 31 Mar 2023 14:42:33 +0100 Subject: [PATCH 012/288] removed obsolete attributes --- indica/models/plasma.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 5fde7a3b..ec34e0fb 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -211,10 +211,6 @@ def initialize_variables(self, tstart: float, tend: float, dt: float): self.tend = tend self.dt = dt - # Dictionary keeping track of deta use for optimisations - self.optimisation: dict = {} - self.forward_models: dict = {} - # Assign plasma and machine attributes self.machine_R = np.linspace( self.machine_dimensions[0][0], self.machine_dimensions[0][1], 100 From 4a5c5b3d3f91caae4117bc34c8c54d583763ad77 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 31 Mar 2023 16:27:11 +0100 Subject: [PATCH 013/288] Setting up testflow --- tests/unit/models/test_plasma.py | 47 ++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/unit/models/test_plasma.py b/tests/unit/models/test_plasma.py index e69de29b..e2a472c0 100644 --- a/tests/unit/models/test_plasma.py +++ b/tests/unit/models/test_plasma.py @@ -0,0 +1,47 @@ +import numpy as np + +from indica.models.plasma import Plasma +from indica.equilibrium import fake_equilibrium_data, Equilibrium + + +class TestPlasmaInit: + + def setup_class(self): + self.tstart = 0. + self.tend = 0.1 + self.dt = 0.02 + self.plasma = Plasma(tstart=self.tstart, tend=self.tend, dt=self.dt, main_ion="h", + impurities=("c", "ar", "he"), impurity_concentration=(0.01, 0.001, 0.01),) + self.equilibrium_data = fake_equilibrium_data( + tstart=self.tstart, tend=self.tend, dt=self.dt, machine_dims=self.plasma.machine_dimensions + ) + self.equilibrium = Equilibrium(self.equilibrium_data) + self.plasma.set_equilibrium(equilibrium=self.equilibrium) + + def teardown_method(self): + self.plasma.initialize_variables(tstart=self.tstart, tend=self.tend, dt=self.dt) + + def test_plasma_initializes(self): + assert hasattr(self, "plasma") + + def test_plasma_volume_non_zero(self): + _volume = self.plasma.volume + assert len(np.where(_volume > 0)[0]) != 0 + + + +# class TestPlasmaProfiles: +# +# def setup_class(self): +# +# def test_update_profiles +# +# def test_assign_profiles +# +# +# +# +# class TestCacheDependency: +# +# def setup_class(self): +# return From 31f306b5fa74026a40126e63d2e010db59cfd0e0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 31 Mar 2023 16:48:47 +0100 Subject: [PATCH 014/288] more tests --- tests/unit/models/test_plasma.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/unit/models/test_plasma.py b/tests/unit/models/test_plasma.py index e2a472c0..5786ade1 100644 --- a/tests/unit/models/test_plasma.py +++ b/tests/unit/models/test_plasma.py @@ -18,6 +18,10 @@ def setup_class(self): self.equilibrium = Equilibrium(self.equilibrium_data) self.plasma.set_equilibrium(equilibrium=self.equilibrium) + def setup_method(self): + self.plasma.electron_density = 1 # set profiles + return + def teardown_method(self): self.plasma.initialize_variables(tstart=self.tstart, tend=self.tend, dt=self.dt) @@ -28,7 +32,19 @@ def test_plasma_volume_non_zero(self): _volume = self.plasma.volume assert len(np.where(_volume > 0)[0]) != 0 + def test_fz_is_non_zero(self): + _fz = self.plasma.fz + assert len(np.where(_fz > 0)[0]) != 0 + + def test_lz_is_non_zero(self): + _lz_tot = self.plasma.lz_tot + assert len(np.where(_lz_tot > 0)[0]) != 0 + + def test_fz_one_time_point(self): + return + def test_fz_keys_match_elements(self): + return # class TestPlasmaProfiles: # @@ -45,3 +61,6 @@ def test_plasma_volume_non_zero(self): # # def setup_class(self): # return +if __name__ == "__main__": + test = TestPlasmaInit() + test.setup_class() From 66e6b22f06488c9355b374296c7ff80538e5f785 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 31 Mar 2023 16:59:05 +0100 Subject: [PATCH 015/288] more test stubs --- tests/unit/models/test_plasma.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/unit/models/test_plasma.py b/tests/unit/models/test_plasma.py index 5786ade1..f1ca96d7 100644 --- a/tests/unit/models/test_plasma.py +++ b/tests/unit/models/test_plasma.py @@ -4,14 +4,16 @@ from indica.equilibrium import fake_equilibrium_data, Equilibrium -class TestPlasmaInit: - +class TestPlasma: def setup_class(self): self.tstart = 0. self.tend = 0.1 self.dt = 0.02 + self.impurities = ("c", "ar", "he") + self.impurity_concentration = (0.01, 0.001, 0.01) + self.plasma = Plasma(tstart=self.tstart, tend=self.tend, dt=self.dt, main_ion="h", - impurities=("c", "ar", "he"), impurity_concentration=(0.01, 0.001, 0.01),) + impurities=self.impurities, impurity_concentration=self.impurity_concentration) self.equilibrium_data = fake_equilibrium_data( tstart=self.tstart, tend=self.tend, dt=self.dt, machine_dims=self.plasma.machine_dimensions ) @@ -28,6 +30,9 @@ def teardown_method(self): def test_plasma_initializes(self): assert hasattr(self, "plasma") + def test_equilibrium_initializes(self): + assert hasattr(self, "equilibrium") + def test_plasma_volume_non_zero(self): _volume = self.plasma.volume assert len(np.where(_volume > 0)[0]) != 0 From b9e61c2d11ee803f9a840dcd7d3da2b90efba7f7 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 12 Apr 2023 09:20:37 +0100 Subject: [PATCH 016/288] small fix --- tests/unit/models/test_plasma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/models/test_plasma.py b/tests/unit/models/test_plasma.py index f1ca96d7..6667e367 100644 --- a/tests/unit/models/test_plasma.py +++ b/tests/unit/models/test_plasma.py @@ -67,5 +67,5 @@ def test_fz_keys_match_elements(self): # def setup_class(self): # return if __name__ == "__main__": - test = TestPlasmaInit() + test = TestPlasma() test.setup_class() From ccfe635c1ddd38bfd99aec8fd42613a4f16d73b6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 25 Apr 2023 14:38:22 +0100 Subject: [PATCH 017/288] cosmetic changes --- indica/models/helike_spectroscopy.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 311aeea7..08ff9fd2 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -693,8 +693,8 @@ def __call__( # fmt: off def doppler_broaden(x, integral, center, ion_mass, ion_temp): - _mass = (ion_mass * constants.proton_mass * constants.c ** 2) - sigma = (np.sqrt(constants.e / _mass * ion_temp) * center) + mass = (ion_mass * constants.proton_mass * constants.c ** 2) + sigma = np.sqrt(constants.e / mass * ion_temp) * center gaussian_broadened = gaussian(x, integral, center, sigma, ) return gaussian_broadened @@ -702,8 +702,6 @@ def doppler_broaden(x, integral, center, ion_mass, ion_temp): def gaussian(x, integral, center, sigma): return (integral / (sigma * np.sqrt(2 * np.pi)) * np.exp(-((x - center) ** 2) / (2 * sigma ** 2))) - - # fmt: on From c8cd11be44d614905a0ad54cb740c08dace37663 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 25 Apr 2023 16:10:24 +0100 Subject: [PATCH 018/288] changed marker style --- indica/workflows/bayes_workflow.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 655eb5ac..449279a8 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -129,14 +129,22 @@ def _plot_1d( zorder=1, color="lightgrey", ) - plt.plot( - diag_data[blobkey].__getattr__(dims[0]), - diag_data[blobkey].sel(t=blob_data.t).values, - linestyle="-", - color="black", - label=f"{blobkey} data", - zorder=4, - ) + if "channel" in dims: + plt.plot( + diag_data[blobkey].__getattr__(dims[0]), + diag_data[blobkey].sel(t=blob_data.t).values, + "k^", + label=f"{blobkey} data", + zorder=4, + ) + else: + plt.plot( + diag_data[blobkey].__getattr__(dims[0]), + diag_data[blobkey].sel(t=blob_data.t).values, + "k-", + label=f"{blobkey} data", + zorder=4, + ) plt.ylabel(ylabel) plt.xlabel(dims[0]) plt.legend() From 5655cef4ffccee8413f1bea172f2ab5861e667dc Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 27 Apr 2023 09:54:57 +0100 Subject: [PATCH 019/288] small fix --- indica/workflows/bayes_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 7a4b6bab..3c0a4836 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -257,7 +257,7 @@ def plot_bayes_result( key, figheader=figheader, filename="temperature", - phantom_profile=phantom_profiles[key], + phantom_profile=phantom_profiles[key].sel(element="ar"), color="red", linestyle="dotted", ) @@ -270,7 +270,7 @@ def plot_bayes_result( blobs[key].sel(element="ar"), key, figheader=figheader, - phantom_profile=phantom_profiles[key], + phantom_profile=phantom_profiles[key].sel(element="ar"), color="red", ) From 40e5cb5bf7a8a5b33a0593a34edff50975224b83 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 27 Apr 2023 11:06:04 +0100 Subject: [PATCH 020/288] adjusting alpha --- indica/workflows/bayes_workflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 3c0a4836..fea6aae8 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -25,7 +25,7 @@ def plot_profile( label=f"{blobkey}, 68% Confidence", zorder=3, color=color, - alpha=0.9, + alpha=0.8, ) plt.fill_between( profile.rho_poloidal, @@ -34,7 +34,7 @@ def plot_profile( label=f"{blobkey}, 95% Confidence", zorder=2, color="grey", - alpha=0.7, + alpha=0.6, ) plt.fill_between( profile.rho_poloidal, @@ -43,7 +43,7 @@ def plot_profile( label=f"{blobkey}, Max-Min", zorder=1, color="lightgrey", - alpha=0.7, + alpha=0.6, ) if phantom_profile is not None: From 210bf7c09194eb3a029893cc3fc2a433507e3162 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 3 May 2023 11:04:07 +0100 Subject: [PATCH 021/288] black reformatting --- indica/models/equilibrium_reconstruction.py | 1 - indica/models/helike_spectroscopy.py | 111 ++++++++++---------- tests/unit/models/test_diagnostic_models.py | 1 - 3 files changed, 55 insertions(+), 58 deletions(-) diff --git a/indica/models/equilibrium_reconstruction.py b/indica/models/equilibrium_reconstruction.py index 01321da4..5d8cae2f 100644 --- a/indica/models/equilibrium_reconstruction.py +++ b/indica/models/equilibrium_reconstruction.py @@ -1,4 +1,3 @@ - import matplotlib.pyplot as plt import xarray as xr diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index f1af47cb..1492531a 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -36,17 +36,17 @@ class Helike_spectroscopy(DiagnosticModel): """ def __init__( - self, - name: str, - instrument_method="get_helike_spectroscopy", - etendue: float = 1.0, - calibration: float = 5.0e-18, - marchuk: bool = True, - full_run: bool = False, - element: str = "ar", - window_len: int = 1030, - window_lim: list = [0.394, 0.401], - window_masks: list = [], + self, + name: str, + instrument_method="get_helike_spectroscopy", + etendue: float = 1.0, + calibration: float = 5.0e-18, + marchuk: bool = True, + full_run: bool = False, + element: str = "ar", + window_len: int = 1030, + window_lim: list = [0.394, 0.401], + window_masks: list = [], ): """ Read all atomic data and initialise objects @@ -159,8 +159,8 @@ def _set_adas_pecs(self): for line in pec: pec[line]["emiss_coeff"] = ( pec[line]["emiss_coeff"] - .sel(electron_density=4.0e19, method="nearest") - .drop_vars("electron_density") + .sel(electron_density=4.0e19, method="nearest") + .drop_vars("electron_density") ) self.pec = pec @@ -231,27 +231,27 @@ def _calculate_line_emission(self, line_labels=["w", "k"]): ) for line_name in line_labels: _line_emission = ( - _pecs.isel( - line_name=(_pecs.wavelength < self.line_ranges[line_name].stop) - & (_pecs.wavelength > self.line_ranges[line_name].start) - ) - * mult + _pecs.isel( + line_name=(_pecs.wavelength < self.line_ranges[line_name].stop) + & (_pecs.wavelength > self.line_ranges[line_name].start) + ) + * mult ) _line_emission = _line_emission.sum(["type", "line_name"]) # TODO: convert PEC wavelengths to nm as per convention at TE! ev_wavelength = ph.nm_eV_conversion(nm=self.line_ranges[line_name].start) line_emission[line_name] = ( - xr.where(_line_emission >= 0, _line_emission, 0) * ev_wavelength - ) * self.calibration + xr.where(_line_emission >= 0, _line_emission, 0) * ev_wavelength + ) * self.calibration if "k" in line_emission.keys() and "w" in line_emission.keys(): line_emission["kw"] = line_emission["k"] * line_emission["w"] if "n3" in line_emission.keys() and "w" in line_emission.keys(): line_emission["n3w"] = line_emission["n345"] * line_emission["w"] if ( - "n3" in line_emission.keys() - and "n345" in line_emission.keys() - and "w" in line_emission.keys() + "n3" in line_emission.keys() + and "n345" in line_emission.keys() + and "w" in line_emission.keys() ): line_emission["tot"] = line_emission["n345"] + line_emission["w"] line_emission["n3tot"] = line_emission["n345"] * line_emission["w"] @@ -319,10 +319,10 @@ def _calculate_temperatures(self): ) def _moment_analysis( - self, - line: str, - profile_1d: DataArray = None, - half_los: bool = True, + self, + line: str, + profile_1d: DataArray = None, + half_los: bool = True, ): """ Perform moment analysis using a specific line emission as distribution function @@ -455,7 +455,7 @@ def _moment_analysis( return result, pos, err_in, err_out def _calculate_intensity( - self, + self, ): """ Returns DataArrays of emission type with co-ordinates of line label and @@ -469,8 +469,8 @@ def _calculate_intensity( _pecs_ds = _pecs.to_dataset("type") temp = [ _pecs_ds[type] - .dropna("line_name", how="all") - .interp(electron_temperature=self.Te, assume_sorted=True) + .dropna("line_name", how="all") + .interp(electron_temperature=self.Te, assume_sorted=True) for type in _pecs_ds.data_vars.keys() ] _intensity = xr.merge(temp).to_array("type") @@ -478,7 +478,7 @@ def _calculate_intensity( return intensity def _make_spectra( - self, + self, ): """ TODO: Doppler Shift / Add convolution of broadening @@ -542,7 +542,9 @@ def _build_bckc_dictionary(self): self.bckc[quant].attrs["emiss"] = self.line_emission[line] elif datatype == ("spectra", "passive"): self.bckc["spectra"] = self.measured_spectra - self.bckc["spectra"].attrs["error"] = np.sqrt(self.measured_spectra) # poisson noise + self.bckc["spectra"].attrs["error"] = np.sqrt( + self.measured_spectra + ) # poisson noise else: print(f"{quant} not available in model for {self.instrument_method}") continue @@ -562,19 +564,19 @@ def _build_bckc_dictionary(self): self.bckc["int_n3/int_tot"] = self.bckc["int_n3"] / self.bckc["int_tot"] def __call__( - self, - Te: DataArray = None, - Ti: DataArray = None, - Ne: DataArray = None, - Nimp: DataArray = None, - Fz: dict = None, - Nh: DataArray = None, - t: LabeledArray = None, - calc_spectra=True, - calc_rho: bool = False, - minimum_lines: bool = False, - moment_analysis: bool = False, - **kwargs, + self, + Te: DataArray = None, + Ti: DataArray = None, + Ne: DataArray = None, + Nimp: DataArray = None, + Fz: dict = None, + Nh: DataArray = None, + t: LabeledArray = None, + calc_spectra=True, + calc_rho: bool = False, + minimum_lines: bool = False, + moment_analysis: bool = False, + **kwargs, ): """ Calculate diagnostic measured values @@ -624,12 +626,12 @@ def __call__( Nimp = self.plasma.impurity_density.interp(t=t) else: if ( - Ne is None - or Te is None - or Nh is None - or Fz is None - or Ti is None - or Nimp is None + Ne is None + or Te is None + or Nh is None + or Fz is None + or Ti is None + or Nimp is None ): raise ValueError("Give inputs or assign plasma class!") @@ -649,10 +651,7 @@ def __call__( quant: dict = {} if moment_analysis: if minimum_lines: - line_labels = [ - "w", - "k" - ] + line_labels = ["w", "k"] names = ["int_w", "int_k", "te_kw", "ti_w"] else: line_labels = list(self.line_ranges.keys()) @@ -664,7 +663,7 @@ def __call__( "te_n3w", "te_kw", "ti_z", - "ti_w" + "ti_w", ] quant = dict(quant, **{x: self.quantities[x] for x in names}) diff --git a/tests/unit/models/test_diagnostic_models.py b/tests/unit/models/test_diagnostic_models.py index 8fa0751d..d0f6bda3 100644 --- a/tests/unit/models/test_diagnostic_models.py +++ b/tests/unit/models/test_diagnostic_models.py @@ -15,7 +15,6 @@ from indica.models.equilibrium_reconstruction import example_run as equil_recon - EXAMPLES: Dict[str, Callable] = { "bolometer_camera": bolo, "diode_filters": diodes, From efe22813d8a3faffdf060a4f9524c723c60e0245 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 3 May 2023 13:38:10 +0100 Subject: [PATCH 022/288] fixing formatting --- indica/bayesmodels.py | 8 ++++++-- tests/unit/models/test_diagnostic_models.py | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index e954e995..40efcc47 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -85,11 +85,15 @@ def _ln_likelihood(self): for key in self.quant_to_optimise: # Float128 since rounding of small numbers causes problems when initial results are bad fits model_data = self.bckc[key].astype("float128") - exp_data = self.data[key].sel(t=self.plasma.time_to_calculate).astype("float128") + exp_data = ( + self.data[key].sel(t=self.plasma.time_to_calculate).astype("float128") + ) exp_error = exp_data * 0.10 # Assume percentage error if none given. if hasattr(self.data[key], "error"): - if (self.data[key].error != 0).any(): # TODO: Some models have an error of 0 given + if ( + self.data[key].error != 0 + ).any(): # TODO: Some models have an error of 0 given exp_error = self.data[key].error _ln_likelihood = np.log(gaussian(model_data, exp_data, exp_error)) diff --git a/tests/unit/models/test_diagnostic_models.py b/tests/unit/models/test_diagnostic_models.py index d0f6bda3..2057e1fd 100644 --- a/tests/unit/models/test_diagnostic_models.py +++ b/tests/unit/models/test_diagnostic_models.py @@ -12,7 +12,6 @@ from indica.models.interferometry import example_run as interf from indica.models.plasma import example_run as example_plasma from indica.models.thomson_scattering import example_run as ts -from indica.models.equilibrium_reconstruction import example_run as equil_recon EXAMPLES: Dict[str, Callable] = { From 7ac95c477ba28c591e9c9c9975dd49ef78ef7df2 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 3 May 2023 13:49:29 +0100 Subject: [PATCH 023/288] flake8 fix --- indica/bayesmodels.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 40efcc47..0d2253f2 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -83,7 +83,8 @@ def _build_bckc(self, params: dict, **kwargs): def _ln_likelihood(self): ln_likelihood = 0 for key in self.quant_to_optimise: - # Float128 since rounding of small numbers causes problems when initial results are bad fits + # Float128 since rounding of small numbers causes problems + # when initial results are bad fits model_data = self.bckc[key].astype("float128") exp_data = ( self.data[key].sel(t=self.plasma.time_to_calculate).astype("float128") From cea61618bd9edef850044debf958ad31e9acff1d Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 3 May 2023 13:54:50 +0100 Subject: [PATCH 024/288] flake8 fix --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 1492531a..0bbefca3 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -280,7 +280,7 @@ def _calculate_los_integral(self, calc_rho=False): t=self.spectra.t, calc_rho=calc_rho, ) - # TODO: LOS integral removes NaNs so manually add them back (find better solution) + # TODO: LOS integral removes NaNs so add them back (find solution) self.measured_spectra[self.measured_spectra == 0] = np.nan def _calculate_temperatures(self): From 0aac384e9951e84dcba9dd56da60db2bd48f6c39 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 3 May 2023 14:30:06 +0100 Subject: [PATCH 025/288] boolean indexing -> xarray.where in call --- indica/models/helike_spectroscopy.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 6df1a992..3d1cfa92 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -280,9 +280,8 @@ def _calculate_los_integral(self, calc_rho=False): t=self.spectra.t, calc_rho=calc_rho, ) - # TODO: LOS integral removes NaNs so add them back (find solution) - self.measured_spectra[self.measured_spectra == 0] = np.nan - + # LOS integral removes NaNs so add them back + self.measured_spectra = self.measured_spectra.where(self.measured_spectra != 0, np.nan) def _calculate_temperatures(self): x1 = self.los_transform.x1 x1_name = self.los_transform.x1_name @@ -771,7 +770,6 @@ def example_run(pulse: int = 9229, plasma=None, plot=False, **kwargs): plasma = example_plasma( pulse=pulse, impurities=("ar",), impurity_concentration=(0.001,), n_rad=10 ) - plasma.time_to_calculate = plasma.t[5] # Create new diagnostic diagnostic_name = "xrcs" los_transform = helike_LOS_example(3) From 0259f22ede1b4c2489d6f5fbcbe2d4322f02e9b0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 3 May 2023 15:24:05 +0100 Subject: [PATCH 026/288] fixing tests --- tests/unit/models/test_plasma.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/unit/models/test_plasma.py b/tests/unit/models/test_plasma.py index 6667e367..201bcdcc 100644 --- a/tests/unit/models/test_plasma.py +++ b/tests/unit/models/test_plasma.py @@ -20,9 +20,9 @@ def setup_class(self): self.equilibrium = Equilibrium(self.equilibrium_data) self.plasma.set_equilibrium(equilibrium=self.equilibrium) - def setup_method(self): - self.plasma.electron_density = 1 # set profiles - return + # def setup_method(self): + # self.plasma.electron_density = 1 # set profiles + # return def teardown_method(self): self.plasma.initialize_variables(tstart=self.tstart, tend=self.tend, dt=self.dt) @@ -37,19 +37,19 @@ def test_plasma_volume_non_zero(self): _volume = self.plasma.volume assert len(np.where(_volume > 0)[0]) != 0 - def test_fz_is_non_zero(self): - _fz = self.plasma.fz - assert len(np.where(_fz > 0)[0]) != 0 + # def test_fz_is_non_zero(self): + # _fz = self.plasma.fz[self.impurities[0]] + # assert len(np.where(_fz > 0)[0]) != 0 + # + # def test_lz_is_non_zero(self): + # _lz_tot = self.plasma.lz_tot[self.impurities[0]] + # assert len(np.where(_lz_tot > 0)[0]) != 0 - def test_lz_is_non_zero(self): - _lz_tot = self.plasma.lz_tot - assert len(np.where(_lz_tot > 0)[0]) != 0 - - def test_fz_one_time_point(self): - return - - def test_fz_keys_match_elements(self): - return + # def test_fz_one_time_point(self): + # return + # + # def test_fz_keys_match_elements(self): + # return # class TestPlasmaProfiles: # From 8cf45671615700bafaa12e57fff9b5042b6af4a0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 3 May 2023 15:33:10 +0100 Subject: [PATCH 027/288] removing import --- tests/integration/models/test_helike_spectroscopy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/models/test_helike_spectroscopy.py b/tests/integration/models/test_helike_spectroscopy.py index 41e58b81..670d7d80 100644 --- a/tests/integration/models/test_helike_spectroscopy.py +++ b/tests/integration/models/test_helike_spectroscopy.py @@ -1,7 +1,6 @@ import indica.models.helike_spectroscopy as helike from indica.models.helike_spectroscopy import helike_LOS_example from indica.models.plasma import example_run as example_plasma -import pytest class TestHelike: def setup_class(self): From 8170cddfd60155e4d4e99cbfe7a1327202964f1f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 3 May 2023 15:52:56 +0100 Subject: [PATCH 028/288] precommit fixes --- indica/models/helike_spectroscopy.py | 6 +++++- .../models/test_helike_spectroscopy.py | 17 +++++++++++---- tests/integration/test_bayesmodels.py | 2 +- tests/unit/models/test_plasma.py | 21 ++++++++++++++----- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 3d1cfa92..6741b17b 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -281,7 +281,10 @@ def _calculate_los_integral(self, calc_rho=False): calc_rho=calc_rho, ) # LOS integral removes NaNs so add them back - self.measured_spectra = self.measured_spectra.where(self.measured_spectra != 0, np.nan) + self.measured_spectra = self.measured_spectra.where( + self.measured_spectra != 0, np.nan + ) + def _calculate_temperatures(self): x1 = self.los_transform.x1 x1_name = self.los_transform.x1_name @@ -764,6 +767,7 @@ def helike_LOS_example(nchannels=3): ) return los_transform + def example_run(pulse: int = 9229, plasma=None, plot=False, **kwargs): # TODO: LOS sometimes crossing bad EFIT reconstruction if plasma is None: diff --git a/tests/integration/models/test_helike_spectroscopy.py b/tests/integration/models/test_helike_spectroscopy.py index 670d7d80..58440694 100644 --- a/tests/integration/models/test_helike_spectroscopy.py +++ b/tests/integration/models/test_helike_spectroscopy.py @@ -2,6 +2,7 @@ from indica.models.helike_spectroscopy import helike_LOS_example from indica.models.plasma import example_run as example_plasma + class TestHelike: def setup_class(self): self.plasma = example_plasma() # using Phantom @@ -16,22 +17,30 @@ def setup_method(self): self.model = helike.Helike_spectroscopy("diagnostic_name") self.model.set_plasma(self.plasma) - def test_helike_moment_runs_with_multiple_LOS(self, ): + def test_helike_moment_runs_with_multiple_LOS( + self, + ): self.model.set_los_transform(self.multiple_channel_los_transform) bckc = self.model(calc_spectra=False, moment_analysis=True) assert bckc - def test_helike_moment_runs_with_single_LOS(self, ): + def test_helike_moment_runs_with_single_LOS( + self, + ): self.model.set_los_transform(self.single_channel_los_transform) bckc = self.model(calc_spectra=False, moment_analysis=True) assert bckc - def test_helike_spectra_with_multiple_LOS(self, ): + def test_helike_spectra_with_multiple_LOS( + self, + ): self.model.set_los_transform(self.multiple_channel_los_transform) bckc = self.model(calc_spectra=True, moment_analysis=False) assert bckc - def test_helike_spectra_with_single_LOS(self, ): + def test_helike_spectra_with_single_LOS( + self, + ): self.model.set_los_transform(self.single_channel_los_transform) bckc = self.model(calc_spectra=True, moment_analysis=False) assert bckc diff --git a/tests/integration/test_bayesmodels.py b/tests/integration/test_bayesmodels.py index 72cb7295..b01f1aca 100644 --- a/tests/integration/test_bayesmodels.py +++ b/tests/integration/test_bayesmodels.py @@ -1,9 +1,9 @@ import emcee import flatdict -from indica.models.helike_spectroscopy import helike_LOS_example from indica.bayesmodels import BayesModels from indica.bayesmodels import get_uniform +from indica.models.helike_spectroscopy import helike_LOS_example from indica.models.helike_spectroscopy import Helike_spectroscopy from indica.models.plasma import example_run diff --git a/tests/unit/models/test_plasma.py b/tests/unit/models/test_plasma.py index 201bcdcc..9b2a8fb7 100644 --- a/tests/unit/models/test_plasma.py +++ b/tests/unit/models/test_plasma.py @@ -1,21 +1,31 @@ import numpy as np +from indica.equilibrium import Equilibrium +from indica.equilibrium import fake_equilibrium_data from indica.models.plasma import Plasma -from indica.equilibrium import fake_equilibrium_data, Equilibrium class TestPlasma: def setup_class(self): - self.tstart = 0. + self.tstart = 0.0 self.tend = 0.1 self.dt = 0.02 self.impurities = ("c", "ar", "he") self.impurity_concentration = (0.01, 0.001, 0.01) - self.plasma = Plasma(tstart=self.tstart, tend=self.tend, dt=self.dt, main_ion="h", - impurities=self.impurities, impurity_concentration=self.impurity_concentration) + self.plasma = Plasma( + tstart=self.tstart, + tend=self.tend, + dt=self.dt, + main_ion="h", + impurities=self.impurities, + impurity_concentration=self.impurity_concentration, + ) self.equilibrium_data = fake_equilibrium_data( - tstart=self.tstart, tend=self.tend, dt=self.dt, machine_dims=self.plasma.machine_dimensions + tstart=self.tstart, + tend=self.tend, + dt=self.dt, + machine_dims=self.plasma.machine_dimensions, ) self.equilibrium = Equilibrium(self.equilibrium_data) self.plasma.set_equilibrium(equilibrium=self.equilibrium) @@ -51,6 +61,7 @@ def test_plasma_volume_non_zero(self): # def test_fz_keys_match_elements(self): # return + # class TestPlasmaProfiles: # # def setup_class(self): From 96f1f5cea7b940df2a2f6007e1eeba9ef15518d2 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 12 May 2023 17:25:28 +0100 Subject: [PATCH 029/288] stashing for quick result --- indica/bayesmodels.py | 16 +- indica/models/helike_spectroscopy.py | 15 +- indica/models/plasma.py | 19 +- indica/workflows/bayes_dev_workflow.py | 334 +++++++++++++++++++++++++ indica/workflows/bayes_workflow.py | 54 +++- 5 files changed, 409 insertions(+), 29 deletions(-) create mode 100644 indica/workflows/bayes_dev_workflow.py diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 0d2253f2..ae3666b5 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -95,10 +95,10 @@ def _ln_likelihood(self): if ( self.data[key].error != 0 ).any(): # TODO: Some models have an error of 0 given - exp_error = self.data[key].error + exp_error = self.data[key].error.sel(t=self.plasma.time_to_calculate) _ln_likelihood = np.log(gaussian(model_data, exp_data, exp_error)) - # Treat channel as key dim which isn't averaged like other dims + # treat channel as key dim which isn't averaged like other dims if "channel" in _ln_likelihood.dims: _ln_likelihood = _ln_likelihood.sum(dim="channel", skipna=True) ln_likelihood += _ln_likelihood.mean(skipna=True).values @@ -186,6 +186,8 @@ def ln_posterior(self, parameters: dict, **kwargs): return -np.inf, {} self.plasma.update_profiles(parameters) + self.plasma.calc_impurity_density("c") # Temp: put this somewhere better + self._build_bckc(parameters, **kwargs) # model calls ln_likelihood = self._ln_likelihood() # compare results to data ln_posterior = ln_likelihood + ln_prior @@ -203,6 +205,16 @@ def ln_posterior(self, parameters: dict, **kwargs): "impurity_density": self.plasma.impurity_density.sel( t=self.plasma.time_to_calculate ), + "ion_density": self.plasma.ion_density.sel( + t=self.plasma.time_to_calculate + ), + "fast_density": self.plasma.fast_density.sel( + t=self.plasma.time_to_calculate + ), + "neutral_density": self.plasma.neutral_density.sel( + t=self.plasma.time_to_calculate + ), + # TODO: add Nh } blob = deepcopy({**self.bckc, **kin_profs}) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 6741b17b..6711bee6 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -40,12 +40,13 @@ def __init__( name: str, instrument_method="get_helike_spectroscopy", etendue: float = 1.0, - calibration: float = 5.0e-18, + calibration: float = 5.0e-19, marchuk: bool = True, full_run: bool = False, element: str = "ar", window_len: int = 1030, window_lim: list = [0.394, 0.401], + window_vector=None, window_masks: list = [], ): """ @@ -55,8 +56,6 @@ def __init__( ---------- name String identifier for the spectrometer - fract_abu - dictionary of fractional abundance objects marchuk Use Marchuk PECs instead of ADAS adf15 files @@ -89,7 +88,11 @@ def __init__( self.pecs = self._set_adas_pecs() self.window_masks = window_masks - window = np.linspace(window_lim[0], window_lim[1], window_len) + self.window_vector = window_vector + if self.window_vector is not None: + window = self.window_vector + else: + window = np.linspace(window_lim[0], window_lim[1], window_len) mask = np.zeros(shape=window.shape) if window_masks: for mslice in window_masks: @@ -578,6 +581,7 @@ def __call__( calc_rho: bool = False, minimum_lines: bool = False, moment_analysis: bool = False, + background=None, **kwargs, ): """ @@ -682,11 +686,14 @@ def __call__( self.intensity = self._calculate_intensity() self.spectra = self._make_spectra() + self._calculate_los_integral( calc_rho=calc_rho, ) if moment_analysis: self._calculate_temperatures() + if background is not None: + self.measured_spectra = self.measured_spectra + background.sel(t=t) self._build_bckc_dictionary() return self.bckc diff --git a/indica/models/plasma.py b/indica/models/plasma.py index ddf70ce2..3bc847ef 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -843,22 +843,13 @@ def vloop(self): self._vloop.loc[dict(t=t)] = vloop.values return self._vloop - def calc_impurity_density(self, t=None): + def calc_impurity_density(self, elements): """ - Calculate impurity density from concentration + Calculate impurity density from concentration and electron density """ - if t is None: - t = self.t - if type(t) is not LabeledArray: - t = [t] - - profile_shape = self.Nimp_prof.yspl / self.Nimp_prof.yspl.sel(rho_poloidal=0) - for elem in self.impurities: - conc = self.impurity_concentration.sel(element=elem) - for _t in t: - dens_0 = self.electron_density.sel(rho_poloidal=0, t=t) * conc - Nimp = profile_shape * dens_0.sel(t=_t) - self.impurity_density.loc[dict(element=elem, t=_t)] = Nimp.values + for elem in elements: + Nimp = self.electron_density * self.impurity_concentration.sel(element=elem) + self.impurity_density.loc[dict(element=elem,)] = Nimp.values def impose_flat_zeff(self): """ diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_dev_workflow.py new file mode 100644 index 00000000..1001cc6e --- /dev/null +++ b/indica/workflows/bayes_dev_workflow.py @@ -0,0 +1,334 @@ +import emcee +import numpy as np +import xarray as xr +import pandas as pd +import flatdict + +from indica.readers.read_st40 import ReadST40 +from indica.bayesmodels import BayesModels, get_uniform +from indica.workflows.bayes_workflow import plot_bayes_result, sample_with_autocorr + +from indica.models.interferometry import Interferometry +from indica.models.helike_spectroscopy import Helike_spectroscopy +from indica.models.charge_exchange import ChargeExchange +from indica.models.equilibrium_reconstruction import EquilibriumReconstruction +from indica.models.plasma import Plasma +from indica.converters.line_of_sight import LineOfSightTransform + +# global configurations +DEFAULT_PHANTOM_PARAMS = { + "Ne_prof.y0": 5e19, + "Ne_prof.wcenter": 0.4, + "Ne_prof.peaking": 2, + "Ne_prof.y1": 2e18, + "Ne_prof.yend": 1e18, + "Ne_prof.wped": 2, + + "Nimp_prof.y0": 3e16, + "Nimp_prof.y1": 0.5e16, + "Nimp_prof.wcenter": 0.4, + "Nimp_prof.wped": 6, + "Nimp_prof.peaking": 2, + + "Te_prof.y0": 3000, + "Te_prof.wcenter": 0.4, + "Te_prof.wped": 4, + "Te_prof.peaking": 2, + + "Ti_prof.y0": 6000, + "Ti_prof.wcenter": 0.4, + "Ti_prof.wped": 4, + "Ti_prof.peaking": 2, + } + +DEFAULT_PRIORS = { + "Ne_prof.y0": get_uniform(2e19, 4e20), + "Ne_prof.y1": get_uniform(1e18, 1e19), + "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), + "Ne_prof.wped": get_uniform(1, 6), + "Ne_prof.wcenter": get_uniform(0.1, 0.8), + "Ne_prof.peaking": get_uniform(1, 4), + + "Nimp_prof.y0": get_uniform(1e15, 1e17), + "Nimp_prof.y1": get_uniform(1e15, 2e16), + "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where((x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0), + "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), + "Nimp_prof.wped": get_uniform(1, 6), + "Nimp_prof.wcenter": get_uniform(0.1, 0.8), + "Nimp_prof.peaking": get_uniform(1, 4), + + "Te_prof.y0": get_uniform(1000, 5000), + "Te_prof.wped": get_uniform(1, 6), + "Te_prof.wcenter": get_uniform(0.1, 0.6), + "Te_prof.peaking": get_uniform(1, 4), + "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode + + "Ti_prof.y0": get_uniform(1000, 10000), + "Ti_prof.wped": get_uniform(1, 6), + "Ti_prof.wcenter": get_uniform(0.1, 0.6), + "Ti_prof.peaking": get_uniform(1, 4), +} + +OPTIMISED_PARAMS = [ + "Ne_prof.y0", + # "Ne_prof.y1", + "Ne_prof.peaking", + "Ne_prof.wcenter", + "Ne_prof.wped", + + "Nimp_prof.y0", + "Nimp_prof.wcenter", + "Nimp_prof.wped", + "Nimp_prof.y1", + "Nimp_prof.peaking", + + "Te_prof.y0", + "Te_prof.wped", + "Te_prof.wcenter", + "Te_prof.peaking", + + "Ti_prof.y0", + "Ti_prof.wped", + "Ti_prof.wcenter", + "Ti_prof.peaking", +] + +OPTIMISED_QUANTITY = [ + "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne" + ] + +class BayesWorkflow: + def __init__(self, + pulse=None, + result_path="./results/example/", + nwalkers=50, + tstart=0.02, + tend=0.10, + dt=0.01, + tsample=4, + iterations=100, + burn_in=0, + diagnostics=None, + phantom=True, + profiles=None + ): + + self.pulse=pulse + self.tstart = tstart + self.tend = tend + self.dt = dt + self.tsample = tsample + self.result_path = result_path + self.nwalkers = nwalkers + self.iterations = iterations + self.burn_in = burn_in + self.diagnostics = diagnostics + self.phantom = phantom + self.profiles = profiles + + self.plasma = Plasma( + tstart=tstart, + tend=tend, + dt=dt, + main_ion="h", + impurities=("ar","c"), + impurity_concentration=(0.001, 0.04, ), + full_run=False, + n_rad=20, + ) + self.plasma.time_to_calculate = self.plasma.t[tsample] + self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) + self.plasma.build_atomic_data(calc_power_loss=False) + + self.init_fast_particles() + self.read_st40(diagnostics) + self.init_models() + + + if self.phantom: + self.phantom_data() + self.save_profiles() + else: + self.real_data() + + self.bayes_run = BayesModels( + plasma=self.plasma, + data=self.flat_data, + diagnostic_models=[*self.models.values()], + quant_to_optimise=OPTIMISED_QUANTITY, + priors=DEFAULT_PRIORS, + ) + + ndim = len(OPTIMISED_PARAMS) + self.start_points = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=self.nwalkers) + self.move = [(emcee.moves.StretchMove(), 1.0), (emcee.moves.DEMove(), 0.0)] + + self.sampler = emcee.EnsembleSampler( + nwalkers, + ndim, + log_prob_fn=self.bayes_run.ln_posterior, + parameter_names=OPTIMISED_PARAMS, + moves=self.move, + kwargs={"moment_analysis": False, "calc_spectra": True, "minimum_lines": False, + "background":self.flat_data["xrcs.background"]}, + ) + + def init_fast_particles(self): + st40_code = ReadST40(13110009, self.tstart, self.tend, dt=self.dt, tree="astra") + st40_code.get_raw_data("", "astra", "RUN573") + st40_code.bin_data_in_time(["astra"], self.tstart, self.tend, self.dt) + code_data = st40_code.binned_data["astra"] + Nf = code_data["nf"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) * 1.0e19 + self.plasma.fast_density.values = Nf.values + Nn = code_data["nn"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) * 1.0e19 + self.plasma.neutral_density.values = Nn.values + Pblon = code_data["pblon"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + self.plasma.pressure_fast_parallel.values = Pblon.values + Pbper = code_data["pbper"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + self.plasma.pressure_fast_perpendicular.values = Pbper.values + + def read_st40(self, diagnostics=None): + self.ST40_data = ReadST40(self.pulse, tstart=self.tstart, tend=self.tend) + self.ST40_data(diagnostics) + self.plasma.set_equilibrium(self.ST40_data.equilibrium) + + def save_profiles(self): + phantom_profiles = { + "electron_density": self.plasma.electron_density.sel( + t=self.plasma.time_to_calculate + ).copy(), + "impurity_density": self.plasma.impurity_density.sel( + t=self.plasma.time_to_calculate + ).copy(), + "electron_temperature": self.plasma.electron_temperature.sel( + t=self.plasma.time_to_calculate + ).copy(), + "ion_temperature": self.plasma.ion_temperature.sel( + t=self.plasma.time_to_calculate + ).copy(), + } + self.profiles = phantom_profiles + + def init_models(self): + self.models = {} + for diag in self.diagnostics: + if diag == "smmh1": + # los_transform = self.ST40_data.binned_data["smmh1"]["ne"].transform + machine_dims = self.plasma.machine_dimensions + origin = np.array([[-0.38063365, 0.91893092, 0.01]]) + # end = np.array([[0, 0, 0.01]]) + direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) + los_transform = LineOfSightTransform( + origin[:, 0], + origin[:, 1], + origin[:, 2], + direction[:, 0], + direction[:, 1], + direction[:, 2], + name="", + machine_dimensions=machine_dims, + passes=2,) + los_transform.set_equilibrium(self.plasma.equilibrium) + model = Interferometry(name="smmh1") + model.set_los_transform(los_transform) + model.plasma = self.plasma + self.models["smmh1"] = model + + if "xrcs" in self.diagnostics: + los_transform = self.ST40_data.binned_data["xrcs"]["te_kw"].transform + model = Helike_spectroscopy(name="xrcs", window_masks=[slice(0.394, 0.396)], + window_vector=self.ST40_data.binned_data["xrcs"]["spectra"].wavelength.values * 0.1) + model.set_los_transform(los_transform) + model.plasma = self.plasma + self.models["xrcs"] = model + + if "efit" in self.diagnostics: + model = EquilibriumReconstruction(name="efit") + model.plasma = self.plasma + self.models["efit"] = model + + if "cxff_pi" in self.diagnostics: + transform = self.ST40_data.binned_data["cxff_pi"]["ti"].transform + transform.set_equilibrium(self.ST40_data.equilibrium) + model = ChargeExchange(name="cxff_pi", element="ar") + model.set_transect_transform(transform) + model.plasma = self.plasma + self.models["cxff_pi"] = model + + + def phantom_data(self, noise=False, noise_factor=0.1): + + self.flat_data = {} + self.flat_data["smmh1.ne"] = self.models["smmh1"]().pop("ne").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["xrcs.spectra"] = self.models["xrcs"]().pop("spectra").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["xrcs.background"] = None + cxrs_data = self.models["cxff_pi"]().pop("ti").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) + self.flat_data["efit.wp"] = self.models["efit"]().pop("wp").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + + if noise: + self.flat_data["smmh1.ne"] = self.flat_data["smmh1.ne"] + self.flat_data["smmh1.ne"].max().values * np.random.normal(0, noise_factor, None) + self.flat_data["xrcs.spectra"] = self.flat_data["xrcs.spectra"] + np.random.normal(0, np.sqrt(self.flat_data["xrcs.spectra"].values[0,]), self.flat_data["xrcs.spectra"].shape[1]) + self.flat_data["cxff_pi.ti"] = self.flat_data["cxff_pi.ti"] + self.flat_data["cxff_pi.ti"].max().values * np.random.normal(0, noise_factor, self.flat_data["cxff_pi.ti"].shape[1]) + self.flat_data["efit.wp"] = self.flat_data["efit.wp"] + self.flat_data["efit.wp"].max().values * np.random.normal(0, noise_factor, None) + + + def real_data(self): + + self.flat_data = flatdict.FlatDict(self.ST40_data.binned_data, ".") + if "xrcs" in self.diagnostics: + self.flat_data["xrcs.spectra"]["wavelength"] = self.flat_data["xrcs.spectra"].wavelength * 0.1 + background = self.flat_data["xrcs.spectra"].where( + (self.flat_data["xrcs.spectra"].wavelength < 0.392) & + (self.flat_data["xrcs.spectra"].wavelength > 0.388), + drop=True) + self.flat_data["xrcs.background"] = background.mean(dim="wavelength") + self.flat_data["xrcs.spectra"]["error"] = np.sqrt(self.flat_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2) + + if "cxff_pi" in self.diagnostics: + self.flat_data["cxff_pi"]["ti"] = self.flat_data["cxff_pi"]["ti"].where(self.flat_data["cxff_pi"]["ti"].channel==2, drop=True) + + + def __call__(self, *args, **kwargs): + + autocorr = sample_with_autocorr( + self.sampler, self.start_points, iterations=self.iterations, auto_sample=5 + ) + blobs = self.sampler.get_blobs(discard=self.burn_in, flat=True) + blob_names = self.sampler.get_blobs().flatten()[0].keys() + blob_dict = { + blob_name: xr.concat( + [data[blob_name] for data in blobs], + dim=pd.Index(np.arange(0, blobs.__len__()), name="index"), + ) + for blob_name in blob_names + } + samples = self.sampler.get_chain(flat=True) + + prior_samples = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=int(1e5)) + result = { + "blobs": blob_dict, + "diag_data": self.flat_data, + "samples": samples, + "prior_samples": prior_samples, + "param_names": OPTIMISED_PARAMS, + "phantom_profiles": self.profiles, + "autocorr": autocorr, + } + self.acceptance_fraction = self.sampler.acceptance_fraction.sum() + print(self.acceptance_fraction) + plot_bayes_result(**result, figheader=self.result_path) + +if __name__ == "__main__": + + run = BayesWorkflow(pulse=10009, result_path="./results/10009_full/", iterations=10000, nwalkers=200, + burn_in=100, diagnostics=[ + "xrcs", + "efit", + "smmh1", + "cxff_pi" + ], phantom=False) + run() diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index fea6aae8..b12e1986 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -47,7 +47,11 @@ def plot_profile( ) if phantom_profile is not None: - phantom_profile.plot( + if "element" in phantom_profile[blobkey].dims: + phantom = phantom_profile[blobkey].sel(element="ar") + else: + phantom = phantom_profile[blobkey] + phantom.plot( label=f"{blobkey}, phantom profile", linestyle=linestyle, color="black", @@ -73,11 +77,12 @@ def _plot_0d( figheader="./results/test/", xlabel="samples ()", ylabel="a.u.", + figsize=(6.4, 4.8), **kwargs, ): if blobkey not in blobs.keys(): raise ValueError(f"{blobkey} not in blobs") - plt.figure() + plt.figure(figsize=figsize) blob_data = blobs[blobkey] plt.plot(blob_data, label=f"{blobkey} model") plt.axhline( @@ -100,12 +105,14 @@ def _plot_1d( filename: str, figheader="./results/test/", ylabel="a.u.", + xlim = None, + figsize=(6.4, 4.8), **kwargs, ): if blobkey not in blobs.keys(): raise ValueError(f"{blobkey} not in blobs") - plt.figure() + plt.figure(figsize=figsize) blob_data = blobs[blobkey] dims = tuple(name for name in blob_data.dims if name != "index") plt.fill_between( @@ -150,6 +157,7 @@ def _plot_1d( ) plt.ylabel(ylabel) plt.xlabel(dims[0]) + plt.xlim(xlim) plt.legend() plt.savefig(figheader + filename) plt.close() @@ -229,8 +237,10 @@ def plot_bayes_result( f"{key.replace('.', '_')}.png", figheader=figheader, ylabel="intensity (A.U.)", + xlim = (0.394, 0.401), + figsize=(12, 10), ) - key = "cxrs.ti" + key = "cxff_pi.ti" if key in blobs.keys(): _plot_1d( blobs, @@ -246,7 +256,7 @@ def plot_bayes_result( blobs[key], key, figheader=figheader, - phantom_profile=phantom_profiles[key], + phantom_profile=phantom_profiles, sharefig=True, color="blue", linestyle="dashdot", @@ -257,22 +267,48 @@ def plot_bayes_result( key, figheader=figheader, filename="temperature", - phantom_profile=phantom_profiles[key].sel(element="ar"), + phantom_profile=phantom_profiles, color="red", linestyle="dotted", ) key = "electron_density" plot_profile( - blobs[key], key, figheader=figheader, phantom_profile=phantom_profiles[key] + blobs[key], key, figheader=figheader, phantom_profile=phantom_profiles ) key = "impurity_density" + for elem in blobs[key].element.values: + plot_profile( + blobs[key].sel(element=elem), + key, + figheader=figheader, + filename=f"{elem} density", + phantom_profile=phantom_profiles, + color="red", + ) + key = "ion_density" plot_profile( - blobs[key].sel(element="ar"), + blobs[key].sel(element="h"), key, figheader=figheader, - phantom_profile=phantom_profiles[key].sel(element="ar"), + filename=f"h density", + phantom_profile=phantom_profiles, color="red", ) + key = "fast_density" + plot_profile( + blobs[key], + key, + figheader=figheader, + phantom_profile=phantom_profiles, + color="red", + ) + key = "neutral_density" + plot_profile( + blobs[key], + key, + figheader=figheader, + phantom_profile=phantom_profiles, + ) corner.corner(samples, labels=param_names) plt.savefig(figheader + "posterior.png") From 7e1fda10f78fd12d0530c417919f440eb6173870 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 May 2023 10:40:32 +0100 Subject: [PATCH 030/288] added calc_impurity for generating profiles from concentrations to update profiles method --- indica/models/plasma.py | 162 ++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 80 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 3bc847ef..83373f7b 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -103,18 +103,18 @@ class Plasma: def __init__( - self, - tstart: float = 0.01, - tend: float = 0.14, - dt: float = 0.01, - machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), - impurities: tuple = ("c", "ar"), - main_ion: str = "h", - impurity_concentration: tuple = (0.02, 0.001), - pulse: int = None, - full_run: bool = False, - n_rad: int = 41, - verbose: bool = False, + self, + tstart: float = 0.01, + tend: float = 0.14, + dt: float = 0.01, + machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), + impurities: tuple = ("c", "ar"), + main_ion: str = "h", + impurity_concentration: tuple = (0.02, 0.001), + pulse: int = None, + full_run: bool = False, + n_rad: int = 41, + verbose: bool = False, ): """ Class for plasma objects. @@ -458,10 +458,12 @@ def initialize_variables(self, tstart: float, tend: float, dt: float): ) def assign_profiles( - self, profile: str = "electron_density", t: float = None, element: str = "ar" + self, profile: str = "electron_density", t: float = None, element: str = "ar" ): if profile == "electron_density": self.electron_density.loc[dict(t=t)] = self.Ne_prof() + self.calc_impurity_density( + tuple([imp for imp in self.impurities if imp != element])) # only update non-argon elif profile == "electron_temperature": self.electron_temperature.loc[dict(t=t)] = self.Te_prof() elif profile == "ion_temperature": @@ -478,15 +480,15 @@ def assign_profiles( ) def update_profiles( - self, - parameters: dict, - profile_prefixs: list = [ - "Te_prof", - "Ti_prof", - "Ne_prof", - "Nimp_prof", - "Vrot_prof", - ], + self, + parameters: dict, + profile_prefixs: list = [ + "Te_prof", + "Ti_prof", + "Ne_prof", + "Nimp_prof", + "Vrot_prof", + ], ): """ Update plasma profiles with profile parameters i.e. @@ -550,7 +552,7 @@ def pressure_tot(self): def pressure_fast(self): # TODO: check whether degrees of freedom are correctly included... self._pressure_fast.values = ( - self.pressure_fast_parallel / 3 + self.pressure_fast_perpendicular * 2 / 3 + self.pressure_fast_parallel / 3 + self.pressure_fast_perpendicular * 2 / 3 ) return self._pressure_fast @@ -617,8 +619,8 @@ def calc_zeff(self): meanz = self.meanz for elem in self.elements: self._zeff.loc[dict(element=elem)] = ( - (ion_density.sel(element=elem) * meanz.sel(element=elem) ** 2) - / electron_density + (ion_density.sel(element=elem) * meanz.sel(element=elem) ** 2) + / electron_density ).values return self._zeff @@ -662,8 +664,8 @@ def calc_lz_tot(self): self.power_loss_tot[elem]( Te, Fz, Ne=Ne, Nh=Nh, full_run=self.full_run ) - .transpose() - .values + .transpose() + .values ) return self._lz_tot @@ -690,8 +692,8 @@ def calc_lz_sxr(self): self.power_loss_sxr[elem]( Te, Fz, Ne=Ne, Nh=Nh, full_run=self.full_run ) - .transpose() - .values + .transpose() + .values ) return self._lz_sxr @@ -704,9 +706,9 @@ def calc_total_radiation(self): ion_density = self.ion_density for elem in self.elements: total_radiation = ( - lz_tot[elem].sum("ion_charges") - * self.electron_density - * ion_density.sel(element=elem) + lz_tot[elem].sum("ion_charges") + * self.electron_density + * ion_density.sel(element=elem) ) self._total_radiation.loc[dict(element=elem)] = xr.where( total_radiation >= 0, @@ -727,9 +729,9 @@ def calc_sxr_radiation(self): ion_density = self.ion_density for elem in self.elements: sxr_radiation = ( - lz_sxr[elem].sum("ion_charges") - * self.electron_density - * ion_density.sel(element=elem) + lz_sxr[elem].sum("ion_charges") + * self.electron_density + * ion_density.sel(element=elem) ) self._sxr_radiation.loc[dict(element=elem)] = xr.where( sxr_radiation >= 0, @@ -849,7 +851,7 @@ def calc_impurity_density(self, elements): """ for elem in elements: Nimp = self.electron_density * self.impurity_concentration.sel(element=elem) - self.impurity_density.loc[dict(element=elem,)] = Nimp.values + self.impurity_density.loc[dict(element=elem, )] = Nimp.values def impose_flat_zeff(self): """ @@ -859,14 +861,14 @@ def impose_flat_zeff(self): for elem in self.impurities: if np.count_nonzero(self.ion_density.sel(element=elem)) != 0: zeff_tmp = ( - self.ion_density.sel(element=elem) - * self.meanz.sel(element=elem) ** 2 - / self.electron_density + self.ion_density.sel(element=elem) + * self.meanz.sel(element=elem) ** 2 + / self.electron_density ) value = zeff_tmp.where(zeff_tmp.rho_poloidal < 0.2).mean("rho_poloidal") zeff_tmp = zeff_tmp / zeff_tmp * value ion_density_tmp = zeff_tmp / ( - self.meanz.sel(element=elem) ** 2 / self.electron_density + self.meanz.sel(element=elem) ** 2 / self.electron_density ) self.ion_density.loc[dict(element=elem)] = ion_density_tmp.values @@ -878,13 +880,13 @@ def convert_in_time(self, value: DataArray, method="linear"): return binned def build_atomic_data( - self, - Te: DataArray = None, - Ne: DataArray = None, - Nh: DataArray = None, - tau: DataArray = None, - default=True, - calc_power_loss=True, + self, + Te: DataArray = None, + Ne: DataArray = None, + Nh: DataArray = None, + tau: DataArray = None, + default=True, + calc_power_loss=True, ): if default: xend = 1.02 @@ -928,8 +930,8 @@ def build_atomic_data( power_loss_tot[elem] = PowerLoss(plt, prb, PRC=prc) power_loss_tot[elem](Te, F_z_t, Ne=Ne, Nh=Nh, full_run=self.full_run) if ( - "pls" in self.adf11[elem].keys() - and "prs" in self.adf11[elem].keys() + "pls" in self.adf11[elem].keys() + and "prs" in self.adf11[elem].keys() ): try: pls = self.ADASReader.get_adf11( @@ -995,13 +997,13 @@ def map_to_midplane(self): for t in np.array(self.time_to_calculate, ndmin=1): rho = ( self.equilibrium.rho.sel(t=t, method="nearest") - .interp(R=R, z=z) - .drop_vars(["R", "z"]) + .interp(R=R, z=z) + .drop_vars(["R", "z"]) ) midplane_profiles[k].append( prof_rho.sel(t=t, method="nearest") - .interp(rho_poloidal=rho) - .drop_vars("rho_poloidal") + .interp(rho_poloidal=rho) + .drop_vars("rho_poloidal") ) midplane_profiles[k] = xr.concat(midplane_profiles[k], "t").assign_coords( t=self.t @@ -1013,7 +1015,7 @@ def map_to_midplane(self): self.midplane_profiles = midplane_profiles def calc_centrifugal_asymmetry( - self, time=None, test_toroidal_rotation=None, plot=False + self, time=None, test_toroidal_rotation=None, plot=False ): """ Calculate (R, z) maps of the ion densities caused by centrifugal asymmetry @@ -1072,12 +1074,12 @@ def calc_centrifugal_asymmetry( self.centrifugal_asymmetry.loc[dict(element=elem)] = asymm asymmetry_factor = asymm.interp(rho_poloidal=self.rho_2d) self.asymmetry_multiplier.loc[dict(element=elem)] = np.exp( - asymmetry_factor * (self.rho_2d.R**2 - R_0**2) + asymmetry_factor * (self.rho_2d.R ** 2 - R_0 ** 2) ) self.ion_density_2d = ( - ion_density.interp(rho_poloidal=self.rho_2d).drop_vars("rho_poloidal") - * self.asymmetry_multiplier + ion_density.interp(rho_poloidal=self.rho_2d).drop_vars("rho_poloidal") + * self.asymmetry_multiplier ) assign_datatype(self.ion_density_2d, ("density", "ion"), "m^-3") @@ -1118,9 +1120,9 @@ def calc_rad_power_2d(self): """ for elem in self.elements: total_radiation = ( - self.lz_tot[elem].sum("ion_charges") - * self.electron_density - * self.ion_density.sel(element=elem) + self.lz_tot[elem].sum("ion_charges") + * self.electron_density + * self.ion_density.sel(element=elem) ) total_radiation = xr.where( total_radiation >= 0, @@ -1130,9 +1132,9 @@ def calc_rad_power_2d(self): self.total_radiation.loc[dict(element=elem)] = total_radiation.values sxr_radiation = ( - self.lz_sxr[elem].sum("ion_charges") - * self.electron_density - * self.ion_density.sel(element=elem) + self.lz_sxr[elem].sum("ion_charges") + * self.electron_density + * self.ion_density.sel(element=elem) ) sxr_radiation = xr.where( sxr_radiation >= 0, @@ -1169,9 +1171,9 @@ def write_to_pickle(self): # Generalized dependency caching class TrackDependecies: def __init__( - self, - operator: Callable, - dependencies: list, + self, + operator: Callable, + dependencies: list, ): """ Call operator only if dependencies variables have changed. @@ -1188,8 +1190,8 @@ def __init__( self.dependencies = dependencies def numpyhash( - self, - nparray: np.array, + self, + nparray: np.array, ): a = nparray.view(np.uint8) return hashlib.sha1(a).hexdigest() @@ -1231,18 +1233,18 @@ def __call__(self): def example_run( - pulse: int = None, - tstart=0.02, - tend=0.1, - dt=0.01, - main_ion="h", - impurities=("c", "ar", "he"), - impurity_concentration=(0.03, 0.001, 0.01), - verbose: bool = True, - n_rad: int = 41, - full_run=False, - calc_power_loss: bool = False, - **kwargs, + pulse: int = None, + tstart=0.02, + tend=0.1, + dt=0.01, + main_ion="h", + impurities=("c", "ar", "he"), + impurity_concentration=(0.03, 0.001, 0.01), + verbose: bool = True, + n_rad: int = 41, + full_run=False, + calc_power_loss: bool = False, + **kwargs, ): # TODO: swap all profiles to new version! From 402cd2d26416ac5cd32e8a108e77909d188b3364 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 May 2023 10:58:23 +0100 Subject: [PATCH 031/288] added conditional handling of diagnostics in data pipeline --- indica/workflows/bayes_dev_workflow.py | 50 +++++++++++++++----------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_dev_workflow.py index 1001cc6e..114ed166 100644 --- a/indica/workflows/bayes_dev_workflow.py +++ b/indica/workflows/bayes_dev_workflow.py @@ -47,7 +47,7 @@ "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), "Ne_prof.wped": get_uniform(1, 6), "Ne_prof.wcenter": get_uniform(0.1, 0.8), - "Ne_prof.peaking": get_uniform(1, 4), + "Ne_prof.peaking": get_uniform(1, 6), "Nimp_prof.y0": get_uniform(1e15, 1e17), "Nimp_prof.y1": get_uniform(1e15, 2e16), @@ -55,23 +55,23 @@ "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), "Nimp_prof.wped": get_uniform(1, 6), "Nimp_prof.wcenter": get_uniform(0.1, 0.8), - "Nimp_prof.peaking": get_uniform(1, 4), + "Nimp_prof.peaking": get_uniform(1, 6), "Te_prof.y0": get_uniform(1000, 5000), "Te_prof.wped": get_uniform(1, 6), "Te_prof.wcenter": get_uniform(0.1, 0.6), - "Te_prof.peaking": get_uniform(1, 4), + "Te_prof.peaking": get_uniform(1, 6), "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode - "Ti_prof.y0": get_uniform(1000, 10000), + "Ti_prof.y0": get_uniform(2000, 10000), "Ti_prof.wped": get_uniform(1, 6), "Ti_prof.wcenter": get_uniform(0.1, 0.6), - "Ti_prof.peaking": get_uniform(1, 4), + "Ti_prof.peaking": get_uniform(1, 6), } OPTIMISED_PARAMS = [ "Ne_prof.y0", - # "Ne_prof.y1", + "Ne_prof.y1", "Ne_prof.peaking", "Ne_prof.wcenter", "Ne_prof.wped", @@ -116,7 +116,7 @@ def __init__(self, profiles=None ): - self.pulse=pulse + self.pulse = pulse self.tstart = tstart self.tend = tend self.dt = dt @@ -134,7 +134,7 @@ def __init__(self, tend=tend, dt=dt, main_ion="h", - impurities=("ar","c"), + impurities=("ar", "c"), impurity_concentration=(0.001, 0.04, ), full_run=False, n_rad=20, @@ -152,7 +152,7 @@ def __init__(self, self.phantom_data() self.save_profiles() else: - self.real_data() + self.exp_data() self.bayes_run = BayesModels( plasma=self.plasma, @@ -164,7 +164,7 @@ def __init__(self, ndim = len(OPTIMISED_PARAMS) self.start_points = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=self.nwalkers) - self.move = [(emcee.moves.StretchMove(), 1.0), (emcee.moves.DEMove(), 0.0)] + self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] self.sampler = emcee.EnsembleSampler( nwalkers, @@ -209,6 +209,12 @@ def save_profiles(self): "ion_temperature": self.plasma.ion_temperature.sel( t=self.plasma.time_to_calculate ).copy(), + "fast_density": self.plasma.fast_density.sel( + t=self.plasma.time_to_calculate + ).copy(), + "neutral_density": self.plasma.neutral_density.sel( + t=self.plasma.time_to_calculate + ).copy(), } self.profiles = phantom_profiles @@ -260,14 +266,17 @@ def init_models(self): def phantom_data(self, noise=False, noise_factor=0.1): - self.flat_data = {} - self.flat_data["smmh1.ne"] = self.models["smmh1"]().pop("ne").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - self.flat_data["xrcs.spectra"] = self.models["xrcs"]().pop("spectra").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - self.flat_data["xrcs.background"] = None - cxrs_data = self.models["cxff_pi"]().pop("ti").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - self.flat_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) - self.flat_data["efit.wp"] = self.models["efit"]().pop("wp").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + if "smmh1" in self.diagnostics: + self.flat_data["smmh1.ne"] = self.models["smmh1"]().pop("ne").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + if "xrcs" in self.diagnostics: + self.flat_data["xrcs.spectra"] = self.models["xrcs"]().pop("spectra").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["xrcs.background"] = None + if "cxff_pi" in self.diagnostics: + cxrs_data = self.models["cxff_pi"]().pop("ti").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) + if "efit" in self.diagnostics: + self.flat_data["efit.wp"] = self.models["efit"]().pop("wp").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) if noise: self.flat_data["smmh1.ne"] = self.flat_data["smmh1.ne"] + self.flat_data["smmh1.ne"].max().values * np.random.normal(0, noise_factor, None) @@ -276,8 +285,7 @@ def phantom_data(self, noise=False, noise_factor=0.1): self.flat_data["efit.wp"] = self.flat_data["efit.wp"] + self.flat_data["efit.wp"].max().values * np.random.normal(0, noise_factor, None) - def real_data(self): - + def exp_data(self): self.flat_data = flatdict.FlatDict(self.ST40_data.binned_data, ".") if "xrcs" in self.diagnostics: self.flat_data["xrcs.spectra"]["wavelength"] = self.flat_data["xrcs.spectra"].wavelength * 0.1 @@ -324,8 +332,8 @@ def __call__(self, *args, **kwargs): if __name__ == "__main__": - run = BayesWorkflow(pulse=10009, result_path="./results/10009_full/", iterations=10000, nwalkers=200, - burn_in=100, diagnostics=[ + run = BayesWorkflow(pulse=10009, result_path="./results/10009_test/", iterations=1000, nwalkers=200, + burn_in=50, diagnostics=[ "xrcs", "efit", "smmh1", From fb7704fe846392de3221011e215f4fd6980ffd71 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 May 2023 10:58:59 +0100 Subject: [PATCH 032/288] moved calc_impurity to plasma method --- indica/bayesmodels.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index ae3666b5..7d56c63e 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -186,8 +186,6 @@ def ln_posterior(self, parameters: dict, **kwargs): return -np.inf, {} self.plasma.update_profiles(parameters) - self.plasma.calc_impurity_density("c") # Temp: put this somewhere better - self._build_bckc(parameters, **kwargs) # model calls ln_likelihood = self._ln_likelihood() # compare results to data ln_posterior = ln_likelihood + ln_prior From 4997194068ca6bf584caa00abe54226503fed964 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 May 2023 15:25:59 +0100 Subject: [PATCH 033/288] tweaking calibration factor --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 6711bee6..74404e1b 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -40,7 +40,7 @@ def __init__( name: str, instrument_method="get_helike_spectroscopy", etendue: float = 1.0, - calibration: float = 5.0e-19, + calibration: float = 5.0e-20, marchuk: bool = True, full_run: bool = False, element: str = "ar", From c8e821f03ec2a032ec3e9f58ee82f53160d3a65e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 May 2023 16:03:16 +0100 Subject: [PATCH 034/288] tweaking calibration factor --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 74404e1b..0c950965 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -40,7 +40,7 @@ def __init__( name: str, instrument_method="get_helike_spectroscopy", etendue: float = 1.0, - calibration: float = 5.0e-20, + calibration: float = 2.0e-20, marchuk: bool = True, full_run: bool = False, element: str = "ar", From 5bb0b516f5e352c95bc10eb166c5b3685d5053e5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 May 2023 17:02:55 +0100 Subject: [PATCH 035/288] tweaking calibration factor --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 0c950965..eae06066 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -40,7 +40,7 @@ def __init__( name: str, instrument_method="get_helike_spectroscopy", etendue: float = 1.0, - calibration: float = 2.0e-20, + calibration: float = 4.0e-20, marchuk: bool = True, full_run: bool = False, element: str = "ar", From 457da6b5e63e728c96ca2cb02ea2267a1283c7da Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 23 May 2023 10:06:08 +0100 Subject: [PATCH 036/288] changed burn_in to burn_in_fraction --- indica/workflows/bayes_dev_workflow.py | 163 +++++++++++++------------ 1 file changed, 88 insertions(+), 75 deletions(-) diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_dev_workflow.py index 114ed166..b7a094a0 100644 --- a/indica/workflows/bayes_dev_workflow.py +++ b/indica/workflows/bayes_dev_workflow.py @@ -4,6 +4,8 @@ import pandas as pd import flatdict +from scipy.stats import loguniform + from indica.readers.read_st40 import ReadST40 from indica.bayesmodels import BayesModels, get_uniform from indica.workflows.bayes_workflow import plot_bayes_result, sample_with_autocorr @@ -17,29 +19,29 @@ # global configurations DEFAULT_PHANTOM_PARAMS = { - "Ne_prof.y0": 5e19, - "Ne_prof.wcenter": 0.4, - "Ne_prof.peaking": 2, - "Ne_prof.y1": 2e18, - "Ne_prof.yend": 1e18, - "Ne_prof.wped": 2, - - "Nimp_prof.y0": 3e16, - "Nimp_prof.y1": 0.5e16, - "Nimp_prof.wcenter": 0.4, - "Nimp_prof.wped": 6, - "Nimp_prof.peaking": 2, - - "Te_prof.y0": 3000, - "Te_prof.wcenter": 0.4, - "Te_prof.wped": 4, - "Te_prof.peaking": 2, - - "Ti_prof.y0": 6000, - "Ti_prof.wcenter": 0.4, - "Ti_prof.wped": 4, - "Ti_prof.peaking": 2, - } + "Ne_prof.y0": 5e19, + "Ne_prof.wcenter": 0.4, + "Ne_prof.peaking": 2, + "Ne_prof.y1": 2e18, + "Ne_prof.yend": 1e18, + "Ne_prof.wped": 2, + + "Nimp_prof.y0": 3e16, + "Nimp_prof.y1": 0.5e16, + "Nimp_prof.wcenter": 0.4, + "Nimp_prof.wped": 6, + "Nimp_prof.peaking": 2, + + "Te_prof.y0": 3000, + "Te_prof.wcenter": 0.4, + "Te_prof.wped": 4, + "Te_prof.peaking": 2, + + "Ti_prof.y0": 6000, + "Ti_prof.wcenter": 0.4, + "Ti_prof.wped": 4, + "Ti_prof.peaking": 2, +} DEFAULT_PRIORS = { "Ne_prof.y0": get_uniform(2e19, 4e20), @@ -49,6 +51,8 @@ "Ne_prof.wcenter": get_uniform(0.1, 0.8), "Ne_prof.peaking": get_uniform(1, 6), + "ar_conc": loguniform(0.0001, 0.01), + "Nimp_prof.y0": get_uniform(1e15, 1e17), "Nimp_prof.y1": get_uniform(1e15, 2e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where((x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0), @@ -56,6 +60,7 @@ "Nimp_prof.wped": get_uniform(1, 6), "Nimp_prof.wcenter": get_uniform(0.1, 0.8), "Nimp_prof.peaking": get_uniform(1, 6), + "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where((x1 > x2), 1, 0), # impurity always more peaked "Te_prof.y0": get_uniform(1000, 5000), "Te_prof.wped": get_uniform(1, 6), @@ -71,34 +76,36 @@ OPTIMISED_PARAMS = [ "Ne_prof.y0", - "Ne_prof.y1", + # "Ne_prof.y1", "Ne_prof.peaking", "Ne_prof.wcenter", "Ne_prof.wped", + # "ar_conc", "Nimp_prof.y0", "Nimp_prof.wcenter", "Nimp_prof.wped", - "Nimp_prof.y1", + # "Nimp_prof.y1", "Nimp_prof.peaking", "Te_prof.y0", - "Te_prof.wped", - "Te_prof.wcenter", - "Te_prof.peaking", + # "Te_prof.wped", + # "Te_prof.wcenter", + # "Te_prof.peaking", "Ti_prof.y0", - "Ti_prof.wped", - "Ti_prof.wcenter", - "Ti_prof.peaking", + # "Ti_prof.wped", + # "Ti_prof.wcenter", + # "Ti_prof.peaking", ] OPTIMISED_QUANTITY = [ - "xrcs.spectra", - "cxff_pi.ti", - "efit.wp", - "smmh1.ne" - ] + "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne" +] + class BayesWorkflow: def __init__(self, @@ -110,7 +117,7 @@ def __init__(self, dt=0.01, tsample=4, iterations=100, - burn_in=0, + burn_in_fraction=0, diagnostics=None, phantom=True, profiles=None @@ -124,7 +131,7 @@ def __init__(self, self.result_path = result_path self.nwalkers = nwalkers self.iterations = iterations - self.burn_in = burn_in + self.burn_in_fraction = burn_in_fraction self.diagnostics = diagnostics self.phantom = phantom self.profiles = profiles @@ -135,7 +142,7 @@ def __init__(self, dt=dt, main_ion="h", impurities=("ar", "c"), - impurity_concentration=(0.001, 0.04, ), + impurity_concentration=(0.001, 0.04,), full_run=False, n_rad=20, ) @@ -147,7 +154,6 @@ def __init__(self, self.read_st40(diagnostics) self.init_models() - if self.phantom: self.phantom_data() self.save_profiles() @@ -155,12 +161,12 @@ def __init__(self, self.exp_data() self.bayes_run = BayesModels( - plasma=self.plasma, - data=self.flat_data, - diagnostic_models=[*self.models.values()], - quant_to_optimise=OPTIMISED_QUANTITY, - priors=DEFAULT_PRIORS, - ) + plasma=self.plasma, + data=self.flat_data, + diagnostic_models=[*self.models.values()], + quant_to_optimise=OPTIMISED_QUANTITY, + priors=DEFAULT_PRIORS, + ) ndim = len(OPTIMISED_PARAMS) self.start_points = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=self.nwalkers) @@ -173,7 +179,7 @@ def __init__(self, parameter_names=OPTIMISED_PARAMS, moves=self.move, kwargs={"moment_analysis": False, "calc_spectra": True, "minimum_lines": False, - "background":self.flat_data["xrcs.background"]}, + "background": self.flat_data["xrcs.background"]}, ) def init_fast_particles(self): @@ -224,9 +230,9 @@ def init_models(self): if diag == "smmh1": # los_transform = self.ST40_data.binned_data["smmh1"]["ne"].transform machine_dims = self.plasma.machine_dimensions - origin = np.array([[-0.38063365, 0.91893092, 0.01]]) + origin = np.array([[-0.38063365, 0.91893092, 0.01]]) # end = np.array([[0, 0, 0.01]]) - direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) + direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) los_transform = LineOfSightTransform( origin[:, 0], origin[:, 1], @@ -236,7 +242,7 @@ def init_models(self): direction[:, 2], name="", machine_dimensions=machine_dims, - passes=2,) + passes=2, ) los_transform.set_equilibrium(self.plasma.equilibrium) model = Interferometry(name="smmh1") model.set_los_transform(los_transform) @@ -246,7 +252,8 @@ def init_models(self): if "xrcs" in self.diagnostics: los_transform = self.ST40_data.binned_data["xrcs"]["te_kw"].transform model = Helike_spectroscopy(name="xrcs", window_masks=[slice(0.394, 0.396)], - window_vector=self.ST40_data.binned_data["xrcs"]["spectra"].wavelength.values * 0.1) + window_vector=self.ST40_data.binned_data["xrcs"][ + "spectra"].wavelength.values * 0.1) model.set_los_transform(los_transform) model.plasma = self.plasma self.models["xrcs"] = model @@ -264,48 +271,54 @@ def init_models(self): model.plasma = self.plasma self.models["cxff_pi"] = model - def phantom_data(self, noise=False, noise_factor=0.1): self.flat_data = {} if "smmh1" in self.diagnostics: - self.flat_data["smmh1.ne"] = self.models["smmh1"]().pop("ne").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["smmh1.ne"] = self.models["smmh1"]().pop("ne").expand_dims( + dim={"t": [self.plasma.time_to_calculate]}) if "xrcs" in self.diagnostics: - self.flat_data["xrcs.spectra"] = self.models["xrcs"]().pop("spectra").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["xrcs.spectra"] = self.models["xrcs"]().pop("spectra").expand_dims( + dim={"t": [self.plasma.time_to_calculate]}) self.flat_data["xrcs.background"] = None if "cxff_pi" in self.diagnostics: cxrs_data = self.models["cxff_pi"]().pop("ti").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) self.flat_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) if "efit" in self.diagnostics: - self.flat_data["efit.wp"] = self.models["efit"]().pop("wp").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["efit.wp"] = self.models["efit"]().pop("wp").expand_dims( + dim={"t": [self.plasma.time_to_calculate]}) if noise: - self.flat_data["smmh1.ne"] = self.flat_data["smmh1.ne"] + self.flat_data["smmh1.ne"].max().values * np.random.normal(0, noise_factor, None) - self.flat_data["xrcs.spectra"] = self.flat_data["xrcs.spectra"] + np.random.normal(0, np.sqrt(self.flat_data["xrcs.spectra"].values[0,]), self.flat_data["xrcs.spectra"].shape[1]) - self.flat_data["cxff_pi.ti"] = self.flat_data["cxff_pi.ti"] + self.flat_data["cxff_pi.ti"].max().values * np.random.normal(0, noise_factor, self.flat_data["cxff_pi.ti"].shape[1]) - self.flat_data["efit.wp"] = self.flat_data["efit.wp"] + self.flat_data["efit.wp"].max().values * np.random.normal(0, noise_factor, None) - + self.flat_data["smmh1.ne"] = self.flat_data["smmh1.ne"] + self.flat_data[ + "smmh1.ne"].max().values * np.random.normal(0, noise_factor, None) + self.flat_data["xrcs.spectra"] = self.flat_data["xrcs.spectra"] + np.random.normal(0, np.sqrt( + self.flat_data["xrcs.spectra"].values[0,]), self.flat_data["xrcs.spectra"].shape[1]) + self.flat_data["cxff_pi.ti"] = self.flat_data["cxff_pi.ti"] + self.flat_data[ + "cxff_pi.ti"].max().values * np.random.normal(0, noise_factor, self.flat_data["cxff_pi.ti"].shape[1]) + self.flat_data["efit.wp"] = self.flat_data["efit.wp"] + self.flat_data[ + "efit.wp"].max().values * np.random.normal(0, noise_factor, None) def exp_data(self): self.flat_data = flatdict.FlatDict(self.ST40_data.binned_data, ".") if "xrcs" in self.diagnostics: self.flat_data["xrcs.spectra"]["wavelength"] = self.flat_data["xrcs.spectra"].wavelength * 0.1 background = self.flat_data["xrcs.spectra"].where( - (self.flat_data["xrcs.spectra"].wavelength < 0.392) & - (self.flat_data["xrcs.spectra"].wavelength > 0.388), - drop=True) + (self.flat_data["xrcs.spectra"].wavelength < 0.392) & + (self.flat_data["xrcs.spectra"].wavelength > 0.388), + drop=True) self.flat_data["xrcs.background"] = background.mean(dim="wavelength") - self.flat_data["xrcs.spectra"]["error"] = np.sqrt(self.flat_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2) + self.flat_data["xrcs.spectra"]["error"] = np.sqrt( + self.flat_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2) if "cxff_pi" in self.diagnostics: - self.flat_data["cxff_pi"]["ti"] = self.flat_data["cxff_pi"]["ti"].where(self.flat_data["cxff_pi"]["ti"].channel==2, drop=True) - + self.flat_data["cxff_pi"]["ti"] = self.flat_data["cxff_pi"]["ti"].where( + self.flat_data["cxff_pi"]["ti"].channel == 2, drop=True) def __call__(self, *args, **kwargs): autocorr = sample_with_autocorr( self.sampler, self.start_points, iterations=self.iterations, auto_sample=5 ) - blobs = self.sampler.get_blobs(discard=self.burn_in, flat=True) + blobs = self.sampler.get_blobs(discard=int(self.iterations * self.burn_in_fraction), flat=True) blob_names = self.sampler.get_blobs().flatten()[0].keys() blob_dict = { blob_name: xr.concat( @@ -330,13 +343,13 @@ def __call__(self, *args, **kwargs): print(self.acceptance_fraction) plot_bayes_result(**result, figheader=self.result_path) -if __name__ == "__main__": - run = BayesWorkflow(pulse=10009, result_path="./results/10009_test/", iterations=1000, nwalkers=200, - burn_in=50, diagnostics=[ - "xrcs", - "efit", - "smmh1", - "cxff_pi" - ], phantom=False) +if __name__ == "__main__": + run = BayesWorkflow(pulse=10009, result_path="./results/10009_test/", iterations=100, nwalkers=50, + burn_in_fraction=0.10, diagnostics=[ + "xrcs", + "efit", + "smmh1", + "cxff_pi" + ], phantom=False) run() From 4b88ffe76b89969e601ff099fc25a4d9c94f0e24 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 23 May 2023 13:36:23 +0100 Subject: [PATCH 037/288] added violinplots --- indica/workflows/bayes_workflow.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index b12e1986..08ed73e4 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -162,6 +162,23 @@ def _plot_1d( plt.savefig(figheader + filename) plt.close() +def violinplots(data, diag_data, filename): + nfig = len(data) + fig, axs = plt.subplots(1, nfig, figsize=(16, 6)) + for idx, key in enumerate(data.keys()): + axs[idx].violinplot(data[key]) + axs[idx].set_xlabel(key) + axs[idx].set_ylabel(data[key].datatype[0]) + axs[idx].axhline( + y=diag_data[key].sel(t=data[key].t).values, + color="black", + linestyle="-", + label=f"{key} data", + ) + + plt.setp([a.get_xticklabels() for a in axs], visible=False) + plt.savefig(filename) + plt.close() def plot_bayes_result( figheader="./results/test/", @@ -188,6 +205,12 @@ def plot_bayes_result( plt.savefig(figheader + "average_tau.png") plt.close() + violinplot_keys = ["efit.wp", "smmh1.ne", "cxff_pi.ti_0d"] + if "cxff_pi.ti" in blobs.keys(): + blobs["cxff_pi.ti_0d"] = blobs["cxff_pi.ti"].sel(channel = diag_data["cxff_pi.ti"].channel) + diag_data["cxff_pi.ti_0d"] = diag_data["cxff_pi.ti"].sel(channel=diag_data["cxff_pi.ti"].channel) + violinplots({key: blobs[key] for key in violinplot_keys if key in blobs.keys()}, diag_data, figheader+"boxplots.png") + key = "efit.wp" if key in blobs.keys(): _plot_0d( From 86e3784f9c9dfbc3f12cacf02d3b1c3912f0f1d2 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 23 May 2023 13:36:49 +0100 Subject: [PATCH 038/288] removed comment --- indica/bayesmodels.py | 1 - 1 file changed, 1 deletion(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 7d56c63e..4ef46d63 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -213,7 +213,6 @@ def ln_posterior(self, parameters: dict, **kwargs): t=self.plasma.time_to_calculate ), - # TODO: add Nh } blob = deepcopy({**self.bckc, **kin_profs}) return ln_posterior, blob From eb13c00d325de6f4c50d9edf188860e1994c0564 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 23 May 2023 13:37:53 +0100 Subject: [PATCH 039/288] included dictionary logic for fitting impurities scaled by concentrations and electron density --- indica/models/plasma.py | 64 ++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 83373f7b..18d0dc08 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -457,43 +457,32 @@ def initialize_variables(self, tstart: float, tend: float, dt: float): ], ) - def assign_profiles( - self, profile: str = "electron_density", t: float = None, element: str = "ar" - ): - if profile == "electron_density": - self.electron_density.loc[dict(t=t)] = self.Ne_prof() - self.calc_impurity_density( - tuple([imp for imp in self.impurities if imp != element])) # only update non-argon - elif profile == "electron_temperature": - self.electron_temperature.loc[dict(t=t)] = self.Te_prof() - elif profile == "ion_temperature": - self.ion_temperature.loc[dict(t=t)] = self.Ti_prof() - elif profile == "toroidal_rotation": - self.toroidal_rotation.loc[dict(t=t)] = self.Vrot_prof() - elif profile == "impurity_density": - self.impurity_density.loc[dict(t=t, element=element)] = self.Nimp_prof() - elif profile == "neutral_density": - self.neutral_density.loc[dict(t=t, element=element)] = self.Nh_prof() - else: - raise ValueError( - f"{profile} currently not found in possible Plasma properties" - ) + def assign_profiles(self, impurity_to_profile=None): + t = self.time_to_calculate + self.electron_density.loc[dict(t=t)] = self.Ne_prof() + self.calc_impurity_density(self.impurities) + self.electron_temperature.loc[dict(t=t)] = self.Te_prof() + self.ion_temperature.loc[dict(t=t)] = self.Ti_prof() + self.toroidal_rotation.loc[dict(t=t)] = self.Vrot_prof() + self.neutral_density.loc[dict(t=t, )] = self.Nh_prof() + if impurity_to_profile is not None: # overwrite impurity profile with non-Ne dependent profile + self.impurity_density.loc[dict(t=t, element=impurity_to_profile)] = self.Nimp_prof() def update_profiles( self, parameters: dict, - profile_prefixs: list = [ - "Te_prof", - "Ti_prof", - "Ne_prof", - "Nimp_prof", - "Vrot_prof", - ], ): """ Update plasma profiles with profile parameters i.e. {"Ne_prof.y0":1e19} -> Ne_prof.y0 """ + profile_prefixs: list = [ + "Te_prof", + "Ti_prof", + "Ne_prof", + "Nimp_prof", + "Vrot_prof", + ] for param, value in parameters.items(): _prefix = [pref for pref in profile_prefixs if pref in param] if _prefix: @@ -505,14 +494,17 @@ def update_profiles( else: raise ValueError(f"parameter: {key} not found in {prefix}") - for key in [ - "electron_density", - "electron_temperature", - "ion_temperature", - "toroidal_rotation", - "impurity_density", - ]: - self.assign_profiles(key, t=self.time_to_calculate) + imp_conc = [key.split("_")[0] for key in parameters.keys() if "conc" in key] + if any(imp_conc): + for idx, key in enumerate(self.impurities): + if any([imp for imp in imp_conc if key in imp]): + self.impurity_concentration[idx] = parameters[key+"_conc"] + + if any([key for key in parameters.keys() if "Nimp_prof" in key]): + impurity_to_profile = "ar" # TODO: generalise this for n independent impurity profiles e.g. Nimp1/Nimp2/... + else: + impurity_to_profile = None + self.assign_profiles(impurity_to_profile=impurity_to_profile) @property def time_to_calculate(self): From 898731150ac91f3ecfb200895b4c4a26570333d9 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 23 May 2023 15:47:43 +0100 Subject: [PATCH 040/288] changed name cxff_pi.ti_0d to cxff_pi.ti0 --- indica/workflows/bayes_workflow.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 08ed73e4..ce82b8d1 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -193,6 +193,7 @@ def plot_bayes_result( ): Path(figheader).mkdir(parents=True, exist_ok=True) + plt.figure() plt.plot( np.arange(0, autocorr.__len__())[np.isfinite(autocorr)], @@ -205,10 +206,10 @@ def plot_bayes_result( plt.savefig(figheader + "average_tau.png") plt.close() - violinplot_keys = ["efit.wp", "smmh1.ne", "cxff_pi.ti_0d"] + violinplot_keys = ["efit.wp", "smmh1.ne", "cxff_pi.ti0"] if "cxff_pi.ti" in blobs.keys(): - blobs["cxff_pi.ti_0d"] = blobs["cxff_pi.ti"].sel(channel = diag_data["cxff_pi.ti"].channel) - diag_data["cxff_pi.ti_0d"] = diag_data["cxff_pi.ti"].sel(channel=diag_data["cxff_pi.ti"].channel) + blobs["cxff_pi.ti0"] = blobs["cxff_pi.ti"].sel(channel = diag_data["cxff_pi.ti"].channel) + diag_data["cxff_pi.ti0"] = diag_data["cxff_pi.ti"].sel(channel=diag_data["cxff_pi.ti"].channel) violinplots({key: blobs[key] for key in violinplot_keys if key in blobs.keys()}, diag_data, figheader+"boxplots.png") key = "efit.wp" From a9b6b18000d06958c29832075c2db2990fd7198d Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 24 May 2023 14:18:49 +0100 Subject: [PATCH 041/288] new option for initialising at maximum-likelihood estimate --- indica/workflows/bayes_dev_workflow.py | 58 +++++++++++++++++++------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_dev_workflow.py index b7a094a0..cb25407a 100644 --- a/indica/workflows/bayes_dev_workflow.py +++ b/indica/workflows/bayes_dev_workflow.py @@ -53,7 +53,7 @@ "ar_conc": loguniform(0.0001, 0.01), - "Nimp_prof.y0": get_uniform(1e15, 1e17), + "Nimp_prof.y0": get_uniform(1e15, 1e18), "Nimp_prof.y1": get_uniform(1e15, 2e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where((x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0), "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), @@ -75,28 +75,28 @@ } OPTIMISED_PARAMS = [ - "Ne_prof.y0", # "Ne_prof.y1", + "Ne_prof.y0", "Ne_prof.peaking", "Ne_prof.wcenter", "Ne_prof.wped", # "ar_conc", + # "Nimp_prof.y1", "Nimp_prof.y0", "Nimp_prof.wcenter", "Nimp_prof.wped", - # "Nimp_prof.y1", "Nimp_prof.peaking", "Te_prof.y0", - # "Te_prof.wped", - # "Te_prof.wcenter", - # "Te_prof.peaking", + "Te_prof.wped", + "Te_prof.wcenter", + "Te_prof.peaking", "Ti_prof.y0", - # "Ti_prof.wped", - # "Ti_prof.wcenter", - # "Ti_prof.peaking", + "Ti_prof.wped", + "Ti_prof.wcenter", + "Ti_prof.peaking", ] OPTIMISED_QUANTITY = [ @@ -118,9 +118,10 @@ def __init__(self, tsample=4, iterations=100, burn_in_fraction=0, + center_mass_sampling = True, diagnostics=None, + profiles=None, phantom=True, - profiles=None ): self.pulse = pulse @@ -132,6 +133,8 @@ def __init__(self, self.nwalkers = nwalkers self.iterations = iterations self.burn_in_fraction = burn_in_fraction + self.center_mass_sampling = center_mass_sampling + self.diagnostics = diagnostics self.phantom = phantom self.profiles = profiles @@ -169,9 +172,7 @@ def __init__(self, ) ndim = len(OPTIMISED_PARAMS) - self.start_points = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=self.nwalkers) self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] - self.sampler = emcee.EnsembleSampler( nwalkers, ndim, @@ -182,6 +183,33 @@ def __init__(self, "background": self.flat_data["xrcs.background"]}, ) + if not self.center_mass_sampling: + self.start_points = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=self.nwalkers) + else: + # gaussian mass around most probable starting points + nsamples = 100 + start_points = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=nsamples) + ln_prob, _ = self.sampler.compute_log_prob(start_points) + num_best_points = int(nsamples * 0.05) + index_best_start = np.argsort(ln_prob)[-num_best_points:] + best_start_points = start_points[index_best_start,:] + best_points_std = np.std(best_start_points, axis=0) + + # Passing samples through ln_prior and redrawing if they fail + samples = np.empty((OPTIMISED_PARAMS.__len__(), 0)) + while samples.size < OPTIMISED_PARAMS.__len__() * self.nwalkers: + sample = np.random.normal(np.mean(best_start_points, axis=0), best_points_std * 2, + size=(self.nwalkers*5, len(OPTIMISED_PARAMS))) + start = {name: sample[:, idx] for idx, name in enumerate(OPTIMISED_PARAMS)} + ln_prior = self.bayes_run._ln_prior(start) + # Convert from dictionary of arrays -> array, + # then filtering out where ln_prior is -infinity + accepted_samples = np.array(list(start.values()))[ + :, ln_prior != -np.inf + ] + samples = np.append(samples, accepted_samples, axis=1) + self.start_points = samples[:, 0:self.nwalkers].transpose() + def init_fast_particles(self): st40_code = ReadST40(13110009, self.tstart, self.tend, dt=self.dt, tree="astra") st40_code.get_raw_data("", "astra", "RUN573") @@ -345,11 +373,11 @@ def __call__(self, *args, **kwargs): if __name__ == "__main__": - run = BayesWorkflow(pulse=10009, result_path="./results/10009_test/", iterations=100, nwalkers=50, - burn_in_fraction=0.10, diagnostics=[ + run = BayesWorkflow(pulse=10009, result_path="./results/10009_test/", iterations=5000, nwalkers=100, + burn_in_fraction=0.20, diagnostics=[ "xrcs", "efit", "smmh1", "cxff_pi" - ], phantom=False) + ], phantom=False, center_mass_sampling=True) run() From 1fa84279ac190e8054092208837029466e0fd7e8 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 24 May 2023 14:19:40 +0100 Subject: [PATCH 042/288] added xlim for violin plot --- indica/workflows/bayes_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index ce82b8d1..25de08be 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -168,6 +168,7 @@ def violinplots(data, diag_data, filename): for idx, key in enumerate(data.keys()): axs[idx].violinplot(data[key]) axs[idx].set_xlabel(key) + axs[idx].set_ylim(bottom=0) axs[idx].set_ylabel(data[key].datatype[0]) axs[idx].axhline( y=diag_data[key].sel(t=data[key].t).values, From db2d0e1f8d77fb4bb239312d7ec31a1ade254140 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 31 May 2023 13:48:51 +0100 Subject: [PATCH 043/288] tweaking calibration --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index eae06066..a0d3aade 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -40,7 +40,7 @@ def __init__( name: str, instrument_method="get_helike_spectroscopy", etendue: float = 1.0, - calibration: float = 4.0e-20, + calibration: float = 8.0e-20, marchuk: bool = True, full_run: bool = False, element: str = "ar", From ce73ee5b823c3a8c3c33a44d9122e1fe42fdfb1e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 31 May 2023 13:50:43 +0100 Subject: [PATCH 044/288] fixed neutral density units --- indica/models/plasma.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 18d0dc08..33014646 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -307,7 +307,7 @@ def initialize_variables(self, tstart: float, tend: float, dt: float): self.data2d, ("density", "electron"), "$m^{-3}$" ) self.neutral_density: DataArray = assign_data( - self.data2d, ("thermal_neutral", "density"), "eV" + self.data2d, ("density", "thermal_neutral"), "$m^{-3}$" ) self.tau: DataArray = assign_data(self.data2d, ("time", "residence"), "s") @@ -462,7 +462,10 @@ def assign_profiles(self, impurity_to_profile=None): self.electron_density.loc[dict(t=t)] = self.Ne_prof() self.calc_impurity_density(self.impurities) self.electron_temperature.loc[dict(t=t)] = self.Te_prof() - self.ion_temperature.loc[dict(t=t)] = self.Ti_prof() + if self.Ti_ref: + self.ion_temperature.loc[dict(t=t)] = self.Ti_prof(y0_ref=self.Te_prof.y0) + else: + self.ion_temperature.loc[dict(t=t)] = self.Ti_prof() self.toroidal_rotation.loc[dict(t=t)] = self.Vrot_prof() self.neutral_density.loc[dict(t=t, )] = self.Nh_prof() if impurity_to_profile is not None: # overwrite impurity profile with non-Ne dependent profile From eda445ae0bd9cfdb576df5aa828f1a0942a06723 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 31 May 2023 13:51:20 +0100 Subject: [PATCH 045/288] added set plot functions --- indica/utilities.py | 95 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/indica/utilities.py b/indica/utilities.py index 585334eb..b8148516 100644 --- a/indica/utilities.py +++ b/indica/utilities.py @@ -1,7 +1,7 @@ """Various miscellanious helper functions.""" - from copy import deepcopy import inspect +import os import string from typing import Any from typing import Callable @@ -12,6 +12,9 @@ from typing import Tuple from typing import Union +from matplotlib import cm +from matplotlib import rcParams +import matplotlib.pylab as plt import numpy as np from scipy.interpolate import CubicSpline from xarray import apply_ufunc @@ -398,3 +401,93 @@ def check_time_present(t_desired: LabeledArray, t_array: LabeledArray): ) if not equil_ok: raise ValueError(f"Desired time {t_desired} not available in array {t_array}") + + +def save_figure( + path_name: str = "", + fig_name: str = "", + orientation: str = "landscape", + dpi: int = 300, + quality: int = 95, + ext: str = "png", + save_fig: bool = True, + create_path: bool = True, +): + if save_fig: + _fig_name = deepcopy(fig_name) + _path_name = deepcopy(path_name) + if _path_name[-1] != "/": + _path_name = f"{_path_name}/" + _file = f"{_path_name}{_fig_name}.{ext}" + + kwargs = {"orientation": orientation, "dpi": dpi} + if ext != "svg": + kwargs["pil_kwargs"] = {"quality": quality} + + if create_path and not os.path.exists(_path_name): + os.mkdir(_path_name) + print(f"Creating directory {_path_name}") + + plt.savefig( + _file, + **kwargs, + ) + print(f"Saving picture to {_file}") + + +def set_plot_colors( + color_map: str = "gnuplot2", +): + + cmap = getattr(cm, color_map) + colors = { + "electron": cmap(0.1), + "ion": cmap(0.75), + "fast": cmap(0.4), + "impurity": cmap(0.0), + "raw": "black", + "thermal": "red", + "total": "black", + "binned": cmap(0.2), + "bckc": cmap(0.75), + } + + return cmap, colors + + +def set_plot_rcparams(option: str = "profiles"): + plot_params: dict = { + "profiles": { + "font.size": 13, + "legend.fontsize": 11, + "lines.markersize": 10, + "lines.linewidth": 2, + }, + "multi": { + "font.size": 13, + "legend.fontsize": 13, + "lines.markersize": 10, + "lines.linewidth": 2, + "figure.figsize": [6.4, 3.8], + "figure.subplot.bottom": 0.15, + }, + "time_evolution": { + "font.size": 11, + "legend.fontsize": 8, + "lines.markersize": 5, + "lines.linewidth": 2, + "font.weight": 600, + }, + } + + if option not in plot_params.keys(): + return + + for key, value in plot_params[option].items(): + rcParams.update({key: value}) + + +def set_axis_sci(axis: str = "y"): + ylim = np.abs(np.array(plt.ylim())) + if ylim.max() > 1.0e3 or ylim.min() < 1.0e-2: + plt.ticklabel_format(style="sci", axis=axis, scilimits=(0, 0)) From 2555605f6db19eef8bd2cbe7e0c7a57fc07ebdbd Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 31 May 2023 13:52:39 +0100 Subject: [PATCH 046/288] added center mass sampling and Ti/Te y0_ref options --- indica/workflows/bayes_dev_workflow.py | 41 +++++++++++++++----------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_dev_workflow.py index cb25407a..ac0110d0 100644 --- a/indica/workflows/bayes_dev_workflow.py +++ b/indica/workflows/bayes_dev_workflow.py @@ -33,13 +33,15 @@ "Nimp_prof.peaking": 2, "Te_prof.y0": 3000, + "Te_prof.y1": 50, "Te_prof.wcenter": 0.4, - "Te_prof.wped": 4, + "Te_prof.wped": 3, "Te_prof.peaking": 2, "Ti_prof.y0": 6000, + "Ti_prof.y1": 50, "Ti_prof.wcenter": 0.4, - "Ti_prof.wped": 4, + "Ti_prof.wped": 3, "Ti_prof.peaking": 2, } @@ -53,13 +55,13 @@ "ar_conc": loguniform(0.0001, 0.01), - "Nimp_prof.y0": get_uniform(1e15, 1e18), + "Nimp_prof.y0": get_uniform(1e16, 1e18), "Nimp_prof.y1": get_uniform(1e15, 2e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where((x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0), "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), "Nimp_prof.wped": get_uniform(1, 6), "Nimp_prof.wcenter": get_uniform(0.1, 0.8), - "Nimp_prof.peaking": get_uniform(1, 6), + "Nimp_prof.peaking": get_uniform(1, 20), "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where((x1 > x2), 1, 0), # impurity always more peaked "Te_prof.y0": get_uniform(1000, 5000), @@ -68,10 +70,10 @@ "Te_prof.peaking": get_uniform(1, 6), "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode - "Ti_prof.y0": get_uniform(2000, 10000), + "Ti_prof.y0": get_uniform(3000, 10000), "Ti_prof.wped": get_uniform(1, 6), "Ti_prof.wcenter": get_uniform(0.1, 0.6), - "Ti_prof.peaking": get_uniform(1, 6), + "Ti_prof.peaking": get_uniform(1, 20), } OPTIMISED_PARAMS = [ @@ -115,10 +117,11 @@ def __init__(self, tstart=0.02, tend=0.10, dt=0.01, - tsample=4, + tsample=0.06, iterations=100, burn_in_fraction=0, - center_mass_sampling = True, + center_mass_sampling=True, + Ti_ref=True, diagnostics=None, profiles=None, phantom=True, @@ -134,6 +137,7 @@ def __init__(self, self.iterations = iterations self.burn_in_fraction = burn_in_fraction self.center_mass_sampling = center_mass_sampling + self.Ti_ref = Ti_ref self.diagnostics = diagnostics self.phantom = phantom @@ -149,7 +153,9 @@ def __init__(self, full_run=False, n_rad=20, ) - self.plasma.time_to_calculate = self.plasma.t[tsample] + self.plasma.Ti_ref = self.Ti_ref + self.tsample = tsample + self.plasma.time_to_calculate = self.plasma.t[np.abs(tsample-self.plasma.t).argmin()] self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) self.plasma.build_atomic_data(calc_power_loss=False) @@ -192,14 +198,14 @@ def __init__(self, ln_prob, _ = self.sampler.compute_log_prob(start_points) num_best_points = int(nsamples * 0.05) index_best_start = np.argsort(ln_prob)[-num_best_points:] - best_start_points = start_points[index_best_start,:] + best_start_points = start_points[index_best_start, :] best_points_std = np.std(best_start_points, axis=0) # Passing samples through ln_prior and redrawing if they fail samples = np.empty((OPTIMISED_PARAMS.__len__(), 0)) while samples.size < OPTIMISED_PARAMS.__len__() * self.nwalkers: sample = np.random.normal(np.mean(best_start_points, axis=0), best_points_std * 2, - size=(self.nwalkers*5, len(OPTIMISED_PARAMS))) + size=(self.nwalkers * 5, len(OPTIMISED_PARAMS))) start = {name: sample[:, idx] for idx, name in enumerate(OPTIMISED_PARAMS)} ln_prior = self.bayes_run._ln_prior(start) # Convert from dictionary of arrays -> array, @@ -225,7 +231,7 @@ def init_fast_particles(self): self.plasma.pressure_fast_perpendicular.values = Pbper.values def read_st40(self, diagnostics=None): - self.ST40_data = ReadST40(self.pulse, tstart=self.tstart, tend=self.tend) + self.ST40_data = ReadST40(self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt) self.ST40_data(diagnostics) self.plasma.set_equilibrium(self.ST40_data.equilibrium) @@ -357,7 +363,7 @@ def __call__(self, *args, **kwargs): } samples = self.sampler.get_chain(flat=True) - prior_samples = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=int(1e5)) + prior_samples = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=int(1e4)) result = { "blobs": blob_dict, "diag_data": self.flat_data, @@ -369,15 +375,16 @@ def __call__(self, *args, **kwargs): } self.acceptance_fraction = self.sampler.acceptance_fraction.sum() print(self.acceptance_fraction) - plot_bayes_result(**result, figheader=self.result_path) + plot_bayes_result(result, figheader=self.result_path, filetype=".png") if __name__ == "__main__": - run = BayesWorkflow(pulse=10009, result_path="./results/10009_test/", iterations=5000, nwalkers=100, - burn_in_fraction=0.20, diagnostics=[ + run = BayesWorkflow(pulse=10009, result_path="./results/10009_60ms_long/", iterations=500, nwalkers=200, + burn_in_fraction=0.10, dt=0.005, tsample=0.060, + diagnostics=[ "xrcs", "efit", "smmh1", "cxff_pi" - ], phantom=False, center_mass_sampling=True) + ], phantom=False, center_mass_sampling=True, Ti_ref=True) run() From 20ae704ee7e8df3e43f6f99587731e9b2de7b56c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 31 May 2023 13:53:07 +0100 Subject: [PATCH 047/288] plot formatting changed --- indica/workflows/bayes_workflow.py | 270 ++++++++++++++++++----------- 1 file changed, 168 insertions(+), 102 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 25de08be..e29add98 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -3,26 +3,46 @@ import corner import matplotlib.pyplot as plt import numpy as np +import pickle + +from indica.utilities import set_plot_rcparams, set_axis_sci def plot_profile( - profile, - blobkey: str, - figheader="./results/test/", - phantom_profile=None, - sharefig=False, - filename="", - linestyle="--", - color="blue", + profile, + blobkey: str, + figheader="./results/test/", + phantom_profile=None, + logscale=False, + sharefig=False, + filename="", + filetype=".png", + linestyle="--", + color="blue", ): - if not plt.get_fignums(): # If no figure is open - plt.figure(figsize=(8, 6)) + set_plot_rcparams("profiles") + # if not plt.get_fignums(): # If no figure is open + # plt.figure(figsize=(8, 6)) + if blobkey == "electron_temperature": + legkey = "Te" + elif blobkey == "ion_temperature": + legkey = "Ti" + elif blobkey == "ion_density": + legkey = "Ni" + elif blobkey == "electron_density": + legkey = "Ne" + elif blobkey == "impurity_density": + legkey = "Nimp" + elif blobkey == "fast_density": + legkey = "Nfast" + elif blobkey == "neutral_density": + legkey = "Nneut" plt.fill_between( profile.rho_poloidal, profile.quantile(0.16, dim="index"), profile.quantile(0.84, dim="index"), - label=f"{blobkey}, 68% Confidence", + label=f"{legkey}, 68% Confidence", zorder=3, color=color, alpha=0.8, @@ -31,7 +51,7 @@ def plot_profile( profile.rho_poloidal, profile.quantile(0.025, dim="index"), profile.quantile(0.975, dim="index"), - label=f"{blobkey}, 95% Confidence", + label=f"{legkey}, 95% Confidence", zorder=2, color="grey", alpha=0.6, @@ -40,7 +60,7 @@ def plot_profile( profile.rho_poloidal, profile.quantile(0.00, dim="index"), profile.quantile(1.00, dim="index"), - label=f"{blobkey}, Max-Min", + label=f"{legkey}, Max-Min", zorder=1, color="lightgrey", alpha=0.6, @@ -52,7 +72,7 @@ def plot_profile( else: phantom = phantom_profile[blobkey] phantom.plot( - label=f"{blobkey}, phantom profile", + label=f"{legkey}, phantom profile", linestyle=linestyle, color="black", zorder=4, @@ -62,52 +82,30 @@ def plot_profile( if sharefig: return + set_axis_sci() + if logscale: + plt.yscale("log") + + plt.xlabel("Rho poloidal") + plt.ylabel(f"{profile.datatype[0].capitalize()} [{profile.units}]") if filename: - plt.savefig(figheader + f"{filename}.png") + plt.savefig(figheader + f"{filename}{filetype}") else: - plt.savefig(figheader + f"{blobkey}.png") + plt.savefig(figheader + f"{blobkey}{filetype}") plt.close("all") -def _plot_0d( - blobs: dict, - blobkey: str, - diag_data: dict, - filename: str, - figheader="./results/test/", - xlabel="samples ()", - ylabel="a.u.", - figsize=(6.4, 4.8), - **kwargs, -): - if blobkey not in blobs.keys(): - raise ValueError(f"{blobkey} not in blobs") - plt.figure(figsize=figsize) - blob_data = blobs[blobkey] - plt.plot(blob_data, label=f"{blobkey} model") - plt.axhline( - y=diag_data[blobkey].sel(t=blob_data.t).values, - color="black", - linestyle="-", - label=f"{blobkey} data", - ) - plt.xlabel(xlabel) - plt.ylabel(ylabel) - plt.legend() - plt.savefig(figheader + filename) - plt.close() - - def _plot_1d( - blobs: dict, - blobkey: str, - diag_data: dict, - filename: str, - figheader="./results/test/", - ylabel="a.u.", - xlim = None, - figsize=(6.4, 4.8), - **kwargs, + blobs: dict, + blobkey: str, + diag_data: dict, + filename: str, + figheader="./results/test/", + ylabel="a.u.", + xlabel="[]", + xlim=None, + figsize=(6.4, 4.8), + **kwargs, ): if blobkey not in blobs.keys(): raise ValueError(f"{blobkey} not in blobs") @@ -155,45 +153,86 @@ def _plot_1d( label=f"{blobkey} data", zorder=4, ) + set_axis_sci() plt.ylabel(ylabel) - plt.xlabel(dims[0]) + plt.xlabel(xlabel) plt.xlim(xlim) plt.legend() plt.savefig(figheader + filename) plt.close() -def violinplots(data, diag_data, filename): + +def violinplot(data: dict, key: str, diag_data: dict, filename: str, ylabel="[a.u.]", figheader="./results/test/", + **kwargs): + set_plot_rcparams() + fig, axs = plt.subplots(1, 1,) + _data = data[key].values + _data = _data[((_data > np.quantile(_data, 0.16)) & (_data < np.quantile(_data, 0.84)))] + violin = axs.violinplot(_data, + showextrema=False, + # quantiles=[0.025, 0.975, 0.16, 0.84], + # showmedians=True, + ) + violin["bodies"][0].set_edgecolor("black") + axs.set_xlabel(key) + top = axs.get_ylim()[1] + axs.set_ylim(bottom=0, top=top * 1.1) + axs.set_ylabel(f"{ylabel}") + y = diag_data[key].sel(t=data[key].t).values + axs.errorbar(1, y=y, yerr=y * 0.10, fmt="D", ecolor="black", capsize=10, color="black") + set_axis_sci() + plt.setp([axs.get_xticklabels()], visible=False) + plt.savefig(figheader + filename) + plt.close() + + +def histograms(data, diag_data, filename): nfig = len(data) fig, axs = plt.subplots(1, nfig, figsize=(16, 6)) for idx, key in enumerate(data.keys()): - axs[idx].violinplot(data[key]) - axs[idx].set_xlabel(key) - axs[idx].set_ylim(bottom=0) - axs[idx].set_ylabel(data[key].datatype[0]) - axs[idx].axhline( - y=diag_data[key].sel(t=data[key].t).values, + n, bins, patches = axs[idx].hist(data[key], 50, density=True) + q1 = (np.percentile(data[key], 16), np.percentile(data[key], 84)) + q2 = (np.percentile(data[key], 2.5), np.percentile(data[key], 97.5)) + idx_high = np.argwhere((bins > q1[0]) & (bins < q1[1])).flatten() + idx_low = np.argwhere((bins > q2[0]) & (bins < q2[1])).flatten() + for patch in patches: + patch.set_facecolor("lightgrey") + for i in idx_low: + patches[i].set_facecolor("grey") + for i in idx_high: + patches[i].set_facecolor("red") + + axs[idx].set_xlabel(f"{key} ({data[key].datatype[0]})") + + axs[idx].axvline( + x=diag_data[key].sel(t=data[key].t).values, color="black", - linestyle="-", + linestyle="-.", label=f"{key} data", ) + axs[0].set_ylabel("pdf ()") - plt.setp([a.get_xticklabels() for a in axs], visible=False) plt.savefig(filename) plt.close() + def plot_bayes_result( - figheader="./results/test/", - blobs=None, - diag_data=None, - samples=None, - prior_samples=None, - param_names=None, - phantom_profiles=None, - autocorr=None, - **kwargs, + results, + figheader="./results/test/", + filetype=".png", + **kwargs, ): - Path(figheader).mkdir(parents=True, exist_ok=True) + diag_data = results["diag_data"] + blobs = results["blobs"] + samples = results["samples"] + prior_samples = results["prior_samples"] + param_names = results["param_names"] + phantom_profiles = results["phantom_profiles"] + autocorr = results["autocorr"] + Path(figheader).mkdir(parents=True, exist_ok=True) + with open(figheader + "results.pkl", "wb") as handle: + pickle.dump(results, handle) plt.figure() plt.plot( @@ -204,54 +243,63 @@ def plot_bayes_result( plt.legend() plt.xlabel("iterations") plt.ylabel("auto-correlation time (iterations)") - plt.savefig(figheader + "average_tau.png") + plt.savefig(figheader + "average_tau" + filetype) plt.close() - violinplot_keys = ["efit.wp", "smmh1.ne", "cxff_pi.ti0"] if "cxff_pi.ti" in blobs.keys(): - blobs["cxff_pi.ti0"] = blobs["cxff_pi.ti"].sel(channel = diag_data["cxff_pi.ti"].channel) + blobs["cxff_pi.ti0"] = blobs["cxff_pi.ti"].sel(channel=diag_data["cxff_pi.ti"].channel) diag_data["cxff_pi.ti0"] = diag_data["cxff_pi.ti"].sel(channel=diag_data["cxff_pi.ti"].channel) - violinplots({key: blobs[key] for key in violinplot_keys if key in blobs.keys()}, diag_data, figheader+"boxplots.png") + + key = "cxff_pi.ti0" + if key in blobs.keys(): + violinplot( + blobs, + key, + diag_data, + f"{key.replace('.', '_')}" + filetype, + figheader=figheader, + ylabel="Temperature [eV]", + ) key = "efit.wp" if key in blobs.keys(): - _plot_0d( + violinplot( blobs, key, diag_data, - f"{key.replace('.', '_')}.png", + f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="Wp (J)", + ylabel="Energy [J]", ) key = "smmh1.ne" if key in blobs.keys(): - _plot_0d( + violinplot( blobs, key, diag_data, - f"{key.replace('.', '_')}.png", + f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="ne_int (m^-2)", + ylabel=r"Line Integrated Density [$m^{-2}$]", ) key = "xrcs.te_kw" if key in blobs.keys(): - _plot_0d( + violinplot( blobs, key, diag_data, - f"{key.replace('.', '_')}.png", + f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="temperature (eV)", + ylabel="Temperature [eV]", ) key = "xrcs.ti_w" if key in blobs.keys(): - _plot_0d( + violinplot( blobs, key, diag_data, - f"{key.replace('.', '_')}.png", + f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="temperature (eV)", + ylabel="Temperature [eV]", ) key = "xrcs.spectra" if key in blobs.keys(): @@ -259,11 +307,12 @@ def plot_bayes_result( blobs, key, diag_data, - f"{key.replace('.', '_')}.png", + f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="intensity (A.U.)", - xlim = (0.394, 0.401), - figsize=(12, 10), + ylabel="Intensity [a.u.]", + xlabel="Wavelength [nm]", + xlim=(0.394, 0.401), + figsize=(6, 4), ) key = "cxff_pi.ti" if key in blobs.keys(): @@ -271,9 +320,11 @@ def plot_bayes_result( blobs, key, diag_data, - f"{key.replace('.', '_')}.png", + f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="temperature (eV)", + ylabel="Temperature [eV]", + xlabel="Channel", + ) key = "electron_temperature" @@ -281,6 +332,7 @@ def plot_bayes_result( blobs[key], key, figheader=figheader, + filetype=filetype, phantom_profile=phantom_profiles, sharefig=True, color="blue", @@ -292,13 +344,14 @@ def plot_bayes_result( key, figheader=figheader, filename="temperature", + filetype=filetype, phantom_profile=phantom_profiles, color="red", linestyle="dotted", ) key = "electron_density" plot_profile( - blobs[key], key, figheader=figheader, phantom_profile=phantom_profiles + blobs[key], key, figheader=figheader, filetype=filetype, phantom_profile=phantom_profiles ) key = "impurity_density" for elem in blobs[key].element.values: @@ -307,6 +360,7 @@ def plot_bayes_result( key, figheader=figheader, filename=f"{elem} density", + filetype=filetype, phantom_profile=phantom_profiles, color="red", ) @@ -316,6 +370,7 @@ def plot_bayes_result( key, figheader=figheader, filename=f"h density", + filetype=filetype, phantom_profile=phantom_profiles, color="red", ) @@ -324,6 +379,7 @@ def plot_bayes_result( blobs[key], key, figheader=figheader, + filetype=filetype, phantom_profile=phantom_profiles, color="red", ) @@ -331,15 +387,18 @@ def plot_bayes_result( plot_profile( blobs[key], key, + filename="", figheader=figheader, + filetype=filetype, phantom_profile=phantom_profiles, + logscale=True ) corner.corner(samples, labels=param_names) - plt.savefig(figheader + "posterior.png") + plt.savefig(figheader + "posterior" + filetype) corner.corner(prior_samples, labels=param_names) - plt.savefig(figheader + "prior.png") + plt.savefig(figheader + "prior" + filetype) plt.close("all") @@ -347,9 +406,9 @@ def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): autocorr = np.ones((iterations,)) * np.nan old_tau = np.inf for sample in sampler.sample( - start_points, - iterations=iterations, - progress=True, + start_points, + iterations=iterations, + progress=True, ): if sampler.iteration % auto_sample: continue @@ -362,3 +421,10 @@ def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): old_tau = new_tau autocorr = autocorr[: sampler.iteration] return autocorr + + +if __name__ == "__main__": + filehead = "./results/10009_60ms_short/" + with open(filehead + "results.pkl", "rb") as handle: + results = pickle.load(handle) + plot_bayes_result(results, filehead, filetype=".png") From e0b9c03518477813d2a060f42f288c18af984346 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 31 May 2023 13:57:44 +0100 Subject: [PATCH 048/288] black formatting --- indica/models/plasma.py | 164 +++++++++--------- indica/workflows/bayes_dev_workflow.py | 227 ++++++++++++++++--------- indica/workflows/bayes_workflow.py | 106 +++++++----- 3 files changed, 294 insertions(+), 203 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 33014646..1f57439c 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -103,18 +103,18 @@ class Plasma: def __init__( - self, - tstart: float = 0.01, - tend: float = 0.14, - dt: float = 0.01, - machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), - impurities: tuple = ("c", "ar"), - main_ion: str = "h", - impurity_concentration: tuple = (0.02, 0.001), - pulse: int = None, - full_run: bool = False, - n_rad: int = 41, - verbose: bool = False, + self, + tstart: float = 0.01, + tend: float = 0.14, + dt: float = 0.01, + machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), + impurities: tuple = ("c", "ar"), + main_ion: str = "h", + impurity_concentration: tuple = (0.02, 0.001), + pulse: int = None, + full_run: bool = False, + n_rad: int = 41, + verbose: bool = False, ): """ Class for plasma objects. @@ -467,13 +467,21 @@ def assign_profiles(self, impurity_to_profile=None): else: self.ion_temperature.loc[dict(t=t)] = self.Ti_prof() self.toroidal_rotation.loc[dict(t=t)] = self.Vrot_prof() - self.neutral_density.loc[dict(t=t, )] = self.Nh_prof() - if impurity_to_profile is not None: # overwrite impurity profile with non-Ne dependent profile - self.impurity_density.loc[dict(t=t, element=impurity_to_profile)] = self.Nimp_prof() + self.neutral_density.loc[ + dict( + t=t, + ) + ] = self.Nh_prof() + if ( + impurity_to_profile is not None + ): # overwrite impurity profile with non-Ne dependent profile + self.impurity_density.loc[ + dict(t=t, element=impurity_to_profile) + ] = self.Nimp_prof() def update_profiles( - self, - parameters: dict, + self, + parameters: dict, ): """ Update plasma profiles with profile parameters i.e. @@ -501,7 +509,7 @@ def update_profiles( if any(imp_conc): for idx, key in enumerate(self.impurities): if any([imp for imp in imp_conc if key in imp]): - self.impurity_concentration[idx] = parameters[key+"_conc"] + self.impurity_concentration[idx] = parameters[key + "_conc"] if any([key for key in parameters.keys() if "Nimp_prof" in key]): impurity_to_profile = "ar" # TODO: generalise this for n independent impurity profiles e.g. Nimp1/Nimp2/... @@ -547,7 +555,7 @@ def pressure_tot(self): def pressure_fast(self): # TODO: check whether degrees of freedom are correctly included... self._pressure_fast.values = ( - self.pressure_fast_parallel / 3 + self.pressure_fast_perpendicular * 2 / 3 + self.pressure_fast_parallel / 3 + self.pressure_fast_perpendicular * 2 / 3 ) return self._pressure_fast @@ -614,8 +622,8 @@ def calc_zeff(self): meanz = self.meanz for elem in self.elements: self._zeff.loc[dict(element=elem)] = ( - (ion_density.sel(element=elem) * meanz.sel(element=elem) ** 2) - / electron_density + (ion_density.sel(element=elem) * meanz.sel(element=elem) ** 2) + / electron_density ).values return self._zeff @@ -659,8 +667,8 @@ def calc_lz_tot(self): self.power_loss_tot[elem]( Te, Fz, Ne=Ne, Nh=Nh, full_run=self.full_run ) - .transpose() - .values + .transpose() + .values ) return self._lz_tot @@ -687,8 +695,8 @@ def calc_lz_sxr(self): self.power_loss_sxr[elem]( Te, Fz, Ne=Ne, Nh=Nh, full_run=self.full_run ) - .transpose() - .values + .transpose() + .values ) return self._lz_sxr @@ -701,9 +709,9 @@ def calc_total_radiation(self): ion_density = self.ion_density for elem in self.elements: total_radiation = ( - lz_tot[elem].sum("ion_charges") - * self.electron_density - * ion_density.sel(element=elem) + lz_tot[elem].sum("ion_charges") + * self.electron_density + * ion_density.sel(element=elem) ) self._total_radiation.loc[dict(element=elem)] = xr.where( total_radiation >= 0, @@ -724,9 +732,9 @@ def calc_sxr_radiation(self): ion_density = self.ion_density for elem in self.elements: sxr_radiation = ( - lz_sxr[elem].sum("ion_charges") - * self.electron_density - * ion_density.sel(element=elem) + lz_sxr[elem].sum("ion_charges") + * self.electron_density + * ion_density.sel(element=elem) ) self._sxr_radiation.loc[dict(element=elem)] = xr.where( sxr_radiation >= 0, @@ -846,7 +854,11 @@ def calc_impurity_density(self, elements): """ for elem in elements: Nimp = self.electron_density * self.impurity_concentration.sel(element=elem) - self.impurity_density.loc[dict(element=elem, )] = Nimp.values + self.impurity_density.loc[ + dict( + element=elem, + ) + ] = Nimp.values def impose_flat_zeff(self): """ @@ -856,14 +868,14 @@ def impose_flat_zeff(self): for elem in self.impurities: if np.count_nonzero(self.ion_density.sel(element=elem)) != 0: zeff_tmp = ( - self.ion_density.sel(element=elem) - * self.meanz.sel(element=elem) ** 2 - / self.electron_density + self.ion_density.sel(element=elem) + * self.meanz.sel(element=elem) ** 2 + / self.electron_density ) value = zeff_tmp.where(zeff_tmp.rho_poloidal < 0.2).mean("rho_poloidal") zeff_tmp = zeff_tmp / zeff_tmp * value ion_density_tmp = zeff_tmp / ( - self.meanz.sel(element=elem) ** 2 / self.electron_density + self.meanz.sel(element=elem) ** 2 / self.electron_density ) self.ion_density.loc[dict(element=elem)] = ion_density_tmp.values @@ -875,13 +887,13 @@ def convert_in_time(self, value: DataArray, method="linear"): return binned def build_atomic_data( - self, - Te: DataArray = None, - Ne: DataArray = None, - Nh: DataArray = None, - tau: DataArray = None, - default=True, - calc_power_loss=True, + self, + Te: DataArray = None, + Ne: DataArray = None, + Nh: DataArray = None, + tau: DataArray = None, + default=True, + calc_power_loss=True, ): if default: xend = 1.02 @@ -925,8 +937,8 @@ def build_atomic_data( power_loss_tot[elem] = PowerLoss(plt, prb, PRC=prc) power_loss_tot[elem](Te, F_z_t, Ne=Ne, Nh=Nh, full_run=self.full_run) if ( - "pls" in self.adf11[elem].keys() - and "prs" in self.adf11[elem].keys() + "pls" in self.adf11[elem].keys() + and "prs" in self.adf11[elem].keys() ): try: pls = self.ADASReader.get_adf11( @@ -992,13 +1004,13 @@ def map_to_midplane(self): for t in np.array(self.time_to_calculate, ndmin=1): rho = ( self.equilibrium.rho.sel(t=t, method="nearest") - .interp(R=R, z=z) - .drop_vars(["R", "z"]) + .interp(R=R, z=z) + .drop_vars(["R", "z"]) ) midplane_profiles[k].append( prof_rho.sel(t=t, method="nearest") - .interp(rho_poloidal=rho) - .drop_vars("rho_poloidal") + .interp(rho_poloidal=rho) + .drop_vars("rho_poloidal") ) midplane_profiles[k] = xr.concat(midplane_profiles[k], "t").assign_coords( t=self.t @@ -1010,7 +1022,7 @@ def map_to_midplane(self): self.midplane_profiles = midplane_profiles def calc_centrifugal_asymmetry( - self, time=None, test_toroidal_rotation=None, plot=False + self, time=None, test_toroidal_rotation=None, plot=False ): """ Calculate (R, z) maps of the ion densities caused by centrifugal asymmetry @@ -1069,12 +1081,12 @@ def calc_centrifugal_asymmetry( self.centrifugal_asymmetry.loc[dict(element=elem)] = asymm asymmetry_factor = asymm.interp(rho_poloidal=self.rho_2d) self.asymmetry_multiplier.loc[dict(element=elem)] = np.exp( - asymmetry_factor * (self.rho_2d.R ** 2 - R_0 ** 2) + asymmetry_factor * (self.rho_2d.R**2 - R_0**2) ) self.ion_density_2d = ( - ion_density.interp(rho_poloidal=self.rho_2d).drop_vars("rho_poloidal") - * self.asymmetry_multiplier + ion_density.interp(rho_poloidal=self.rho_2d).drop_vars("rho_poloidal") + * self.asymmetry_multiplier ) assign_datatype(self.ion_density_2d, ("density", "ion"), "m^-3") @@ -1115,9 +1127,9 @@ def calc_rad_power_2d(self): """ for elem in self.elements: total_radiation = ( - self.lz_tot[elem].sum("ion_charges") - * self.electron_density - * self.ion_density.sel(element=elem) + self.lz_tot[elem].sum("ion_charges") + * self.electron_density + * self.ion_density.sel(element=elem) ) total_radiation = xr.where( total_radiation >= 0, @@ -1127,9 +1139,9 @@ def calc_rad_power_2d(self): self.total_radiation.loc[dict(element=elem)] = total_radiation.values sxr_radiation = ( - self.lz_sxr[elem].sum("ion_charges") - * self.electron_density - * self.ion_density.sel(element=elem) + self.lz_sxr[elem].sum("ion_charges") + * self.electron_density + * self.ion_density.sel(element=elem) ) sxr_radiation = xr.where( sxr_radiation >= 0, @@ -1166,9 +1178,9 @@ def write_to_pickle(self): # Generalized dependency caching class TrackDependecies: def __init__( - self, - operator: Callable, - dependencies: list, + self, + operator: Callable, + dependencies: list, ): """ Call operator only if dependencies variables have changed. @@ -1185,8 +1197,8 @@ def __init__( self.dependencies = dependencies def numpyhash( - self, - nparray: np.array, + self, + nparray: np.array, ): a = nparray.view(np.uint8) return hashlib.sha1(a).hexdigest() @@ -1228,18 +1240,18 @@ def __call__(self): def example_run( - pulse: int = None, - tstart=0.02, - tend=0.1, - dt=0.01, - main_ion="h", - impurities=("c", "ar", "he"), - impurity_concentration=(0.03, 0.001, 0.01), - verbose: bool = True, - n_rad: int = 41, - full_run=False, - calc_power_loss: bool = False, - **kwargs, + pulse: int = None, + tstart=0.02, + tend=0.1, + dt=0.01, + main_ion="h", + impurities=("c", "ar", "he"), + impurity_concentration=(0.03, 0.001, 0.01), + verbose: bool = True, + n_rad: int = 41, + full_run=False, + calc_power_loss: bool = False, + **kwargs, ): # TODO: swap all profiles to new version! diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_dev_workflow.py index ac0110d0..c673d2e7 100644 --- a/indica/workflows/bayes_dev_workflow.py +++ b/indica/workflows/bayes_dev_workflow.py @@ -25,19 +25,16 @@ "Ne_prof.y1": 2e18, "Ne_prof.yend": 1e18, "Ne_prof.wped": 2, - "Nimp_prof.y0": 3e16, "Nimp_prof.y1": 0.5e16, "Nimp_prof.wcenter": 0.4, "Nimp_prof.wped": 6, "Nimp_prof.peaking": 2, - "Te_prof.y0": 3000, "Te_prof.y1": 50, "Te_prof.wcenter": 0.4, "Te_prof.wped": 3, "Te_prof.peaking": 2, - "Ti_prof.y0": 6000, "Ti_prof.y1": 50, "Ti_prof.wcenter": 0.4, @@ -52,24 +49,24 @@ "Ne_prof.wped": get_uniform(1, 6), "Ne_prof.wcenter": get_uniform(0.1, 0.8), "Ne_prof.peaking": get_uniform(1, 6), - "ar_conc": loguniform(0.0001, 0.01), - "Nimp_prof.y0": get_uniform(1e16, 1e18), "Nimp_prof.y1": get_uniform(1e15, 2e16), - "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where((x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0), + "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( + (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 + ), "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), "Nimp_prof.wped": get_uniform(1, 6), "Nimp_prof.wcenter": get_uniform(0.1, 0.8), "Nimp_prof.peaking": get_uniform(1, 20), - "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where((x1 > x2), 1, 0), # impurity always more peaked - + "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where( + (x1 > x2), 1, 0 + ), # impurity always more peaked "Te_prof.y0": get_uniform(1000, 5000), "Te_prof.wped": get_uniform(1, 6), "Te_prof.wcenter": get_uniform(0.1, 0.6), "Te_prof.peaking": get_uniform(1, 6), "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode - "Ti_prof.y0": get_uniform(3000, 10000), "Ti_prof.wped": get_uniform(1, 6), "Ti_prof.wcenter": get_uniform(0.1, 0.6), @@ -82,51 +79,43 @@ "Ne_prof.peaking", "Ne_prof.wcenter", "Ne_prof.wped", - # "ar_conc", # "Nimp_prof.y1", "Nimp_prof.y0", "Nimp_prof.wcenter", "Nimp_prof.wped", "Nimp_prof.peaking", - "Te_prof.y0", "Te_prof.wped", "Te_prof.wcenter", "Te_prof.peaking", - "Ti_prof.y0", "Ti_prof.wped", "Ti_prof.wcenter", "Ti_prof.peaking", ] -OPTIMISED_QUANTITY = [ - "xrcs.spectra", - "cxff_pi.ti", - "efit.wp", - "smmh1.ne" -] +OPTIMISED_QUANTITY = ["xrcs.spectra", "cxff_pi.ti", "efit.wp", "smmh1.ne"] class BayesWorkflow: - def __init__(self, - pulse=None, - result_path="./results/example/", - nwalkers=50, - tstart=0.02, - tend=0.10, - dt=0.01, - tsample=0.06, - iterations=100, - burn_in_fraction=0, - center_mass_sampling=True, - Ti_ref=True, - diagnostics=None, - profiles=None, - phantom=True, - ): - + def __init__( + self, + pulse=None, + result_path="./results/example/", + nwalkers=50, + tstart=0.02, + tend=0.10, + dt=0.01, + tsample=0.06, + iterations=100, + burn_in_fraction=0, + center_mass_sampling=True, + Ti_ref=True, + diagnostics=None, + profiles=None, + phantom=True, + ): self.pulse = pulse self.tstart = tstart self.tend = tend @@ -149,13 +138,18 @@ def __init__(self, dt=dt, main_ion="h", impurities=("ar", "c"), - impurity_concentration=(0.001, 0.04,), + impurity_concentration=( + 0.001, + 0.04, + ), full_run=False, n_rad=20, ) self.plasma.Ti_ref = self.Ti_ref self.tsample = tsample - self.plasma.time_to_calculate = self.plasma.t[np.abs(tsample-self.plasma.t).argmin()] + self.plasma.time_to_calculate = self.plasma.t[ + np.abs(tsample - self.plasma.t).argmin() + ] self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) self.plasma.build_atomic_data(calc_power_loss=False) @@ -185,16 +179,24 @@ def __init__(self, log_prob_fn=self.bayes_run.ln_posterior, parameter_names=OPTIMISED_PARAMS, moves=self.move, - kwargs={"moment_analysis": False, "calc_spectra": True, "minimum_lines": False, - "background": self.flat_data["xrcs.background"]}, + kwargs={ + "moment_analysis": False, + "calc_spectra": True, + "minimum_lines": False, + "background": self.flat_data["xrcs.background"], + }, ) if not self.center_mass_sampling: - self.start_points = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=self.nwalkers) + self.start_points = self.bayes_run.sample_from_priors( + OPTIMISED_PARAMS, size=self.nwalkers + ) else: - # gaussian mass around most probable starting points + # gaussian mass around most probable starting points TODO: move this function to it's own method nsamples = 100 - start_points = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=nsamples) + start_points = self.bayes_run.sample_from_priors( + OPTIMISED_PARAMS, size=nsamples + ) ln_prob, _ = self.sampler.compute_log_prob(start_points) num_best_points = int(nsamples * 0.05) index_best_start = np.argsort(ln_prob)[-num_best_points:] @@ -204,26 +206,37 @@ def __init__(self, # Passing samples through ln_prior and redrawing if they fail samples = np.empty((OPTIMISED_PARAMS.__len__(), 0)) while samples.size < OPTIMISED_PARAMS.__len__() * self.nwalkers: - sample = np.random.normal(np.mean(best_start_points, axis=0), best_points_std * 2, - size=(self.nwalkers * 5, len(OPTIMISED_PARAMS))) - start = {name: sample[:, idx] for idx, name in enumerate(OPTIMISED_PARAMS)} + sample = np.random.normal( + np.mean(best_start_points, axis=0), + best_points_std * 2, + size=(self.nwalkers * 5, len(OPTIMISED_PARAMS)), + ) + start = { + name: sample[:, idx] for idx, name in enumerate(OPTIMISED_PARAMS) + } ln_prior = self.bayes_run._ln_prior(start) # Convert from dictionary of arrays -> array, # then filtering out where ln_prior is -infinity accepted_samples = np.array(list(start.values()))[ - :, ln_prior != -np.inf - ] + :, ln_prior != -np.inf + ] samples = np.append(samples, accepted_samples, axis=1) - self.start_points = samples[:, 0:self.nwalkers].transpose() + self.start_points = samples[:, 0 : self.nwalkers].transpose() def init_fast_particles(self): st40_code = ReadST40(13110009, self.tstart, self.tend, dt=self.dt, tree="astra") st40_code.get_raw_data("", "astra", "RUN573") st40_code.bin_data_in_time(["astra"], self.tstart, self.tend, self.dt) code_data = st40_code.binned_data["astra"] - Nf = code_data["nf"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) * 1.0e19 + Nf = ( + code_data["nf"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + * 1.0e19 + ) self.plasma.fast_density.values = Nf.values - Nn = code_data["nn"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) * 1.0e19 + Nn = ( + code_data["nn"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + * 1.0e19 + ) self.plasma.neutral_density.values = Nn.values Pblon = code_data["pblon"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) self.plasma.pressure_fast_parallel.values = Pblon.values @@ -231,7 +244,9 @@ def init_fast_particles(self): self.plasma.pressure_fast_perpendicular.values = Pbper.values def read_st40(self, diagnostics=None): - self.ST40_data = ReadST40(self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt) + self.ST40_data = ReadST40( + self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt + ) self.ST40_data(diagnostics) self.plasma.set_equilibrium(self.ST40_data.equilibrium) @@ -276,7 +291,8 @@ def init_models(self): direction[:, 2], name="", machine_dimensions=machine_dims, - passes=2, ) + passes=2, + ) los_transform.set_equilibrium(self.plasma.equilibrium) model = Interferometry(name="smmh1") model.set_los_transform(los_transform) @@ -285,9 +301,14 @@ def init_models(self): if "xrcs" in self.diagnostics: los_transform = self.ST40_data.binned_data["xrcs"]["te_kw"].transform - model = Helike_spectroscopy(name="xrcs", window_masks=[slice(0.394, 0.396)], - window_vector=self.ST40_data.binned_data["xrcs"][ - "spectra"].wavelength.values * 0.1) + model = Helike_spectroscopy( + name="xrcs", + window_masks=[slice(0.394, 0.396)], + window_vector=self.ST40_data.binned_data["xrcs"][ + "spectra" + ].wavelength.values + * 0.1, + ) model.set_los_transform(los_transform) model.plasma = self.plasma self.models["xrcs"] = model @@ -308,51 +329,80 @@ def init_models(self): def phantom_data(self, noise=False, noise_factor=0.1): self.flat_data = {} if "smmh1" in self.diagnostics: - self.flat_data["smmh1.ne"] = self.models["smmh1"]().pop("ne").expand_dims( - dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["smmh1.ne"] = ( + self.models["smmh1"]() + .pop("ne") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + ) if "xrcs" in self.diagnostics: - self.flat_data["xrcs.spectra"] = self.models["xrcs"]().pop("spectra").expand_dims( - dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["xrcs.spectra"] = ( + self.models["xrcs"]() + .pop("spectra") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + ) self.flat_data["xrcs.background"] = None if "cxff_pi" in self.diagnostics: - cxrs_data = self.models["cxff_pi"]().pop("ti").expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + cxrs_data = ( + self.models["cxff_pi"]() + .pop("ti") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + ) self.flat_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) if "efit" in self.diagnostics: - self.flat_data["efit.wp"] = self.models["efit"]().pop("wp").expand_dims( - dim={"t": [self.plasma.time_to_calculate]}) + self.flat_data["efit.wp"] = ( + self.models["efit"]() + .pop("wp") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + ) if noise: self.flat_data["smmh1.ne"] = self.flat_data["smmh1.ne"] + self.flat_data[ - "smmh1.ne"].max().values * np.random.normal(0, noise_factor, None) - self.flat_data["xrcs.spectra"] = self.flat_data["xrcs.spectra"] + np.random.normal(0, np.sqrt( - self.flat_data["xrcs.spectra"].values[0,]), self.flat_data["xrcs.spectra"].shape[1]) - self.flat_data["cxff_pi.ti"] = self.flat_data["cxff_pi.ti"] + self.flat_data[ - "cxff_pi.ti"].max().values * np.random.normal(0, noise_factor, self.flat_data["cxff_pi.ti"].shape[1]) + "smmh1.ne" + ].max().values * np.random.normal(0, noise_factor, None) + self.flat_data["xrcs.spectra"] = self.flat_data[ + "xrcs.spectra" + ] + np.random.normal( + 0, + np.sqrt(self.flat_data["xrcs.spectra"].values[0,]), + self.flat_data["xrcs.spectra"].shape[1], + ) + self.flat_data["cxff_pi.ti"] = self.flat_data[ + "cxff_pi.ti" + ] + self.flat_data["cxff_pi.ti"].max().values * np.random.normal( + 0, noise_factor, self.flat_data["cxff_pi.ti"].shape[1] + ) self.flat_data["efit.wp"] = self.flat_data["efit.wp"] + self.flat_data[ - "efit.wp"].max().values * np.random.normal(0, noise_factor, None) + "efit.wp" + ].max().values * np.random.normal(0, noise_factor, None) def exp_data(self): self.flat_data = flatdict.FlatDict(self.ST40_data.binned_data, ".") if "xrcs" in self.diagnostics: - self.flat_data["xrcs.spectra"]["wavelength"] = self.flat_data["xrcs.spectra"].wavelength * 0.1 + self.flat_data["xrcs.spectra"]["wavelength"] = ( + self.flat_data["xrcs.spectra"].wavelength * 0.1 + ) background = self.flat_data["xrcs.spectra"].where( - (self.flat_data["xrcs.spectra"].wavelength < 0.392) & - (self.flat_data["xrcs.spectra"].wavelength > 0.388), - drop=True) + (self.flat_data["xrcs.spectra"].wavelength < 0.392) + & (self.flat_data["xrcs.spectra"].wavelength > 0.388), + drop=True, + ) self.flat_data["xrcs.background"] = background.mean(dim="wavelength") self.flat_data["xrcs.spectra"]["error"] = np.sqrt( - self.flat_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2) + self.flat_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 + ) if "cxff_pi" in self.diagnostics: self.flat_data["cxff_pi"]["ti"] = self.flat_data["cxff_pi"]["ti"].where( - self.flat_data["cxff_pi"]["ti"].channel == 2, drop=True) + self.flat_data["cxff_pi"]["ti"].channel == 2, drop=True + ) def __call__(self, *args, **kwargs): - autocorr = sample_with_autocorr( self.sampler, self.start_points, iterations=self.iterations, auto_sample=5 ) - blobs = self.sampler.get_blobs(discard=int(self.iterations * self.burn_in_fraction), flat=True) + blobs = self.sampler.get_blobs( + discard=int(self.iterations * self.burn_in_fraction), flat=True + ) blob_names = self.sampler.get_blobs().flatten()[0].keys() blob_dict = { blob_name: xr.concat( @@ -363,7 +413,9 @@ def __call__(self, *args, **kwargs): } samples = self.sampler.get_chain(flat=True) - prior_samples = self.bayes_run.sample_from_priors(OPTIMISED_PARAMS, size=int(1e4)) + prior_samples = self.bayes_run.sample_from_priors( + OPTIMISED_PARAMS, size=int(1e4) + ) result = { "blobs": blob_dict, "diag_data": self.flat_data, @@ -379,12 +431,17 @@ def __call__(self, *args, **kwargs): if __name__ == "__main__": - run = BayesWorkflow(pulse=10009, result_path="./results/10009_60ms_long/", iterations=500, nwalkers=200, - burn_in_fraction=0.10, dt=0.005, tsample=0.060, - diagnostics=[ - "xrcs", - "efit", - "smmh1", - "cxff_pi" - ], phantom=False, center_mass_sampling=True, Ti_ref=True) + run = BayesWorkflow( + pulse=10009, + result_path="./results/10009_60ms_long/", + iterations=500, + nwalkers=200, + burn_in_fraction=0.10, + dt=0.005, + tsample=0.060, + diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], + phantom=False, + center_mass_sampling=True, + Ti_ref=True, + ) run() diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index e29add98..781bce8d 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -9,16 +9,16 @@ def plot_profile( - profile, - blobkey: str, - figheader="./results/test/", - phantom_profile=None, - logscale=False, - sharefig=False, - filename="", - filetype=".png", - linestyle="--", - color="blue", + profile, + blobkey: str, + figheader="./results/test/", + phantom_profile=None, + logscale=False, + sharefig=False, + filename="", + filetype=".png", + linestyle="--", + color="blue", ): set_plot_rcparams("profiles") # if not plt.get_fignums(): # If no figure is open @@ -96,16 +96,16 @@ def plot_profile( def _plot_1d( - blobs: dict, - blobkey: str, - diag_data: dict, - filename: str, - figheader="./results/test/", - ylabel="a.u.", - xlabel="[]", - xlim=None, - figsize=(6.4, 4.8), - **kwargs, + blobs: dict, + blobkey: str, + diag_data: dict, + filename: str, + figheader="./results/test/", + ylabel="a.u.", + xlabel="[]", + xlim=None, + figsize=(6.4, 4.8), + **kwargs, ): if blobkey not in blobs.keys(): raise ValueError(f"{blobkey} not in blobs") @@ -162,24 +162,39 @@ def _plot_1d( plt.close() -def violinplot(data: dict, key: str, diag_data: dict, filename: str, ylabel="[a.u.]", figheader="./results/test/", - **kwargs): +def violinplot( + data: dict, + key: str, + diag_data: dict, + filename: str, + ylabel="[a.u.]", + figheader="./results/test/", + **kwargs, +): set_plot_rcparams() - fig, axs = plt.subplots(1, 1,) + fig, axs = plt.subplots( + 1, + 1, + ) _data = data[key].values - _data = _data[((_data > np.quantile(_data, 0.16)) & (_data < np.quantile(_data, 0.84)))] - violin = axs.violinplot(_data, - showextrema=False, - # quantiles=[0.025, 0.975, 0.16, 0.84], - # showmedians=True, - ) + _data = _data[ + ((_data > np.quantile(_data, 0.16)) & (_data < np.quantile(_data, 0.84))) + ] + violin = axs.violinplot( + _data, + showextrema=False, + # quantiles=[0.025, 0.975, 0.16, 0.84], + # showmedians=True, + ) violin["bodies"][0].set_edgecolor("black") axs.set_xlabel(key) top = axs.get_ylim()[1] axs.set_ylim(bottom=0, top=top * 1.1) axs.set_ylabel(f"{ylabel}") y = diag_data[key].sel(t=data[key].t).values - axs.errorbar(1, y=y, yerr=y * 0.10, fmt="D", ecolor="black", capsize=10, color="black") + axs.errorbar( + 1, y=y, yerr=y * 0.10, fmt="D", ecolor="black", capsize=10, color="black" + ) set_axis_sci() plt.setp([axs.get_xticklabels()], visible=False) plt.savefig(figheader + filename) @@ -217,10 +232,10 @@ def histograms(data, diag_data, filename): def plot_bayes_result( - results, - figheader="./results/test/", - filetype=".png", - **kwargs, + results, + figheader="./results/test/", + filetype=".png", + **kwargs, ): diag_data = results["diag_data"] blobs = results["blobs"] @@ -247,8 +262,12 @@ def plot_bayes_result( plt.close() if "cxff_pi.ti" in blobs.keys(): - blobs["cxff_pi.ti0"] = blobs["cxff_pi.ti"].sel(channel=diag_data["cxff_pi.ti"].channel) - diag_data["cxff_pi.ti0"] = diag_data["cxff_pi.ti"].sel(channel=diag_data["cxff_pi.ti"].channel) + blobs["cxff_pi.ti0"] = blobs["cxff_pi.ti"].sel( + channel=diag_data["cxff_pi.ti"].channel + ) + diag_data["cxff_pi.ti0"] = diag_data["cxff_pi.ti"].sel( + channel=diag_data["cxff_pi.ti"].channel + ) key = "cxff_pi.ti0" if key in blobs.keys(): @@ -324,7 +343,6 @@ def plot_bayes_result( figheader=figheader, ylabel="Temperature [eV]", xlabel="Channel", - ) key = "electron_temperature" @@ -351,7 +369,11 @@ def plot_bayes_result( ) key = "electron_density" plot_profile( - blobs[key], key, figheader=figheader, filetype=filetype, phantom_profile=phantom_profiles + blobs[key], + key, + figheader=figheader, + filetype=filetype, + phantom_profile=phantom_profiles, ) key = "impurity_density" for elem in blobs[key].element.values: @@ -391,7 +413,7 @@ def plot_bayes_result( figheader=figheader, filetype=filetype, phantom_profile=phantom_profiles, - logscale=True + logscale=True, ) corner.corner(samples, labels=param_names) @@ -406,9 +428,9 @@ def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): autocorr = np.ones((iterations,)) * np.nan old_tau = np.inf for sample in sampler.sample( - start_points, - iterations=iterations, - progress=True, + start_points, + iterations=iterations, + progress=True, ): if sampler.iteration % auto_sample: continue From dfabdd0bc5eb6dabee45e711365b987b977e9bb1 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 31 May 2023 14:05:54 +0100 Subject: [PATCH 049/288] deleted example bayes_opt --- indica/workflows/example_bayes_opt.py | 180 -------------------------- 1 file changed, 180 deletions(-) delete mode 100644 indica/workflows/example_bayes_opt.py diff --git a/indica/workflows/example_bayes_opt.py b/indica/workflows/example_bayes_opt.py deleted file mode 100644 index 4ac1bcee..00000000 --- a/indica/workflows/example_bayes_opt.py +++ /dev/null @@ -1,180 +0,0 @@ -import emcee -import numpy as np -import pandas as pd -import xarray as xr - -from indica.bayesmodels import BayesModels -from indica.bayesmodels import get_uniform -from indica.models.helike_spectroscopy import Helike_spectroscopy -from indica.models.interferometry import Interferometry -from indica.models.plasma import Plasma -from indica.readers.read_st40 import ReadST40 -from indica.workflows.bayes_workflow import plot_bayes_result -from indica.workflows.bayes_workflow import sample_with_autocorr - -# TODO: allow conditional prior usage even when only -# one param is being optimisied i.e. 1 is constant - - -def run( - pulse, - phantom_profile_params, - iterations, - result_path, - burn_in=0, - tstart=0.02, - tend=0.10, - dt=0.01, - tsample=3, -): - plasma = Plasma( - tstart=tstart, - tend=tend, - dt=dt, - main_ion="h", - impurities=("ar",), - impurity_concentration=(0.001,), - full_run=False, - n_rad=10, - ) - plasma.time_to_calculate = plasma.t[tsample] - plasma.update_profiles(phantom_profile_params) - plasma.build_atomic_data() - # Make phantom profiles - phantom_profiles = { - "electron_density": plasma.Ne_prof.yspl, - "electron_temperature": plasma.Te_prof.yspl, - "ion_temperature": plasma.Ti_prof.yspl, - "impurity_density": plasma.Nimp_prof.yspl, - } - - ST40 = ReadST40(pulse, tstart=tstart, tend=tend) - ST40(["xrcs", "smmh1"]) - - # Initialise Diagnostic Models - los_transform = ST40.binned_data["smmh1"]["ne"].transform - smmh1 = Interferometry(name="smmh1") - smmh1.set_los_transform(los_transform) - smmh1.plasma = plasma - los_transform = ST40.binned_data["xrcs"]["te_kw"].transform - xrcs = Helike_spectroscopy(name="xrcs", window_masks=[slice(0.3945, 0.3962)]) - xrcs.set_los_transform(los_transform) - xrcs.plasma = plasma - - flat_data = {} - flat_data["smmh1.ne"] = ( - smmh1().pop("ne").expand_dims(dim={"t": [plasma.time_to_calculate]}) - ) - flat_data["xrcs.spectra"] = ( - xrcs().pop("spectra").expand_dims(dim={"t": [plasma.time_to_calculate]}) - ) - - priors = { - "Ne_prof.y0": get_uniform(1e19, 8e19), - "Ne_prof.y1": get_uniform(1e18, 5e18), - "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where(((x1 > x2 * 2)), 1, 0), - "Ne_prof.wped": get_uniform(1, 5), - "Ne_prof.wcenter": get_uniform(0.1, 0.8), - "Ne_prof.peaking": get_uniform(1, 5), - "Nimp_prof.peaking": get_uniform(1, 8), - "Nimp_prof.wcenter": get_uniform(0.1, 0.4), - "Nimp_prof.y0": get_uniform(1e16, 5e16), - "Nimp_prof.y1": get_uniform(1e15, 1e16), - "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( - (x1 > x2 * 100) & (x1 < x2 * 1e4), 1, 0 - ), - "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), - "Te_prof.y0": get_uniform(1000, 6000), - "Te_prof.peaking": get_uniform(1, 4), - "Ti_prof.y0": get_uniform(2000, 10000), - "Ti_prof.peaking": get_uniform(1, 4), - } - # Setup Optimiser - param_names = [ - "Ne_prof.y0", - # "Ne_prof.y1", - # "Ne_prof.peaking", - "Nimp_prof.y0", - # "Nimp_prof.y1", - # "Nimp_prof.peaking", - "Te_prof.y0", - # "Te_prof.peaking", - "Ti_prof.y0", - # "Ti_prof.peaking", - ] - - bm = BayesModels( - plasma=plasma, - data=flat_data, - diagnostic_models=[smmh1, xrcs], - quant_to_optimise=[ - "smmh1.ne", - "xrcs.spectra", - ], - priors=priors, - ) - - ndim = param_names.__len__() - nwalkers = 20 - start_points = bm.sample_from_priors(param_names, size=nwalkers) - move = [(emcee.moves.StretchMove(), 1.0), (emcee.moves.DEMove(), 0.0)] - - sampler = emcee.EnsembleSampler( - nwalkers, - ndim, - log_prob_fn=bm.ln_posterior, - parameter_names=param_names, - moves=move, - kwargs={"moment_analysis": False, "calc_spectra": True}, - ) - - autocorr = sample_with_autocorr( - sampler, start_points, iterations=iterations, auto_sample=5 - ) - - blobs = sampler.get_blobs(discard=burn_in, flat=True) - blob_names = sampler.get_blobs().flatten()[0].keys() - blob_dict = { - blob_name: xr.concat( - [data[blob_name] for data in blobs], - dim=pd.Index(np.arange(0, blobs.__len__()), name="index"), - ) - for blob_name in blob_names - } - samples = sampler.get_chain(flat=True) - - prior_samples = bm.sample_from_priors(param_names, size=int(1e5)) - - # TODO make sure xrcs bckc doesn't have dims t and channels - # save result - result = { - "blobs": blob_dict, - "diag_data": flat_data, - "samples": samples, - "prior_samples": prior_samples, - "param_names": param_names, - "phantom_profiles": phantom_profiles, - "plasma": plasma, - "autocorr": autocorr, - } - print(sampler.acceptance_fraction.sum()) - plot_bayes_result(**result, figheader=result_path) - - -if __name__ == "__main__": - params = { - "Ne_prof.y0": 5e19, - "Ne_prof.wcenter": 0.4, - "Ne_prof.peaking": 2, - "Ne_prof.y1": 2e18, - "Ne_prof.yend": 1e18, - "Ne_prof.wped": 2, - "Nimp_prof.y0": 2e16, - "Nimp_prof.y1": 2e15, - "Nimp_prof.peaking": 2, - "Te_prof.y0": 3000, - "Te_prof.peaking": 2, - "Ti_prof.y0": 5000, - "Ti_prof.peaking": 2, - } - run(10009, params, 10, "./results/test/", burn_in=0) From b21b66d2007d793b8da970b83e955f391d2e164a Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 31 May 2023 14:07:40 +0100 Subject: [PATCH 050/288] moved sample with autocorr function to bayes_dev_workflow --- indica/workflows/bayes_dev_workflow.py | 23 ++++++++++++++++++++++- indica/workflows/bayes_workflow.py | 21 --------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_dev_workflow.py index c673d2e7..a0e4eaba 100644 --- a/indica/workflows/bayes_dev_workflow.py +++ b/indica/workflows/bayes_dev_workflow.py @@ -8,7 +8,7 @@ from indica.readers.read_st40 import ReadST40 from indica.bayesmodels import BayesModels, get_uniform -from indica.workflows.bayes_workflow import plot_bayes_result, sample_with_autocorr +from indica.workflows.bayes_workflow import plot_bayes_result from indica.models.interferometry import Interferometry from indica.models.helike_spectroscopy import Helike_spectroscopy @@ -430,6 +430,27 @@ def __call__(self, *args, **kwargs): plot_bayes_result(result, figheader=self.result_path, filetype=".png") +def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): + autocorr = np.ones((iterations,)) * np.nan + old_tau = np.inf + for sample in sampler.sample( + start_points, + iterations=iterations, + progress=True, + ): + if sampler.iteration % auto_sample: + continue + new_tau = sampler.get_autocorr_time(tol=0) + autocorr[sampler.iteration - 1] = np.mean(new_tau) + converged = np.all(new_tau * 50 < sampler.iteration) + converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) + if converged: + break + old_tau = new_tau + autocorr = autocorr[: sampler.iteration] + return autocorr + + if __name__ == "__main__": run = BayesWorkflow( pulse=10009, diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 781bce8d..69ed7a46 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -424,27 +424,6 @@ def plot_bayes_result( plt.close("all") -def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): - autocorr = np.ones((iterations,)) * np.nan - old_tau = np.inf - for sample in sampler.sample( - start_points, - iterations=iterations, - progress=True, - ): - if sampler.iteration % auto_sample: - continue - new_tau = sampler.get_autocorr_time(tol=0) - autocorr[sampler.iteration - 1] = np.mean(new_tau) - converged = np.all(new_tau * 50 < sampler.iteration) - converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) - if converged: - break - old_tau = new_tau - autocorr = autocorr[: sampler.iteration] - return autocorr - - if __name__ == "__main__": filehead = "./results/10009_60ms_short/" with open(filehead + "results.pkl", "rb") as handle: From 2f78d9d745ebab776f42eada4e9ed4da3261fb5b Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 31 May 2023 14:08:53 +0100 Subject: [PATCH 051/288] renamed bayesworkflow --- indica/workflows/bayes_dev_workflow.py | 2 +- indica/workflows/{bayes_workflow.py => bayes_plots.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename indica/workflows/{bayes_workflow.py => bayes_plots.py} (100%) diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_dev_workflow.py index a0e4eaba..f17e1c6e 100644 --- a/indica/workflows/bayes_dev_workflow.py +++ b/indica/workflows/bayes_dev_workflow.py @@ -8,7 +8,7 @@ from indica.readers.read_st40 import ReadST40 from indica.bayesmodels import BayesModels, get_uniform -from indica.workflows.bayes_workflow import plot_bayes_result +from indica.workflows.bayes_plots import plot_bayes_result from indica.models.interferometry import Interferometry from indica.models.helike_spectroscopy import Helike_spectroscopy diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_plots.py similarity index 100% rename from indica/workflows/bayes_workflow.py rename to indica/workflows/bayes_plots.py From 9ed488ea0ec750717f5c5d1657fab2df275ce1df Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 13 Jun 2023 09:40:12 +0100 Subject: [PATCH 052/288] stashing --- indica/workflows/bayes_dev_workflow.py | 94 ++++++++++++++++++++------ indica/workflows/bayes_plots.py | 80 +++++++++++----------- 2 files changed, 116 insertions(+), 58 deletions(-) diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_dev_workflow.py index f17e1c6e..d6e72780 100644 --- a/indica/workflows/bayes_dev_workflow.py +++ b/indica/workflows/bayes_dev_workflow.py @@ -3,13 +3,13 @@ import xarray as xr import pandas as pd import flatdict - from scipy.stats import loguniform +from pathlib import Path +import pickle from indica.readers.read_st40 import ReadST40 from indica.bayesmodels import BayesModels, get_uniform from indica.workflows.bayes_plots import plot_bayes_result - from indica.models.interferometry import Interferometry from indica.models.helike_spectroscopy import Helike_spectroscopy from indica.models.charge_exchange import ChargeExchange @@ -17,6 +17,8 @@ from indica.models.plasma import Plasma from indica.converters.line_of_sight import LineOfSightTransform +from indica.writers.bda_tree import create_nodes, write_nodes + # global configurations DEFAULT_PHANTOM_PARAMS = { "Ne_prof.y0": 5e19, @@ -82,17 +84,17 @@ # "ar_conc", # "Nimp_prof.y1", "Nimp_prof.y0", - "Nimp_prof.wcenter", - "Nimp_prof.wped", - "Nimp_prof.peaking", + # "Nimp_prof.wcenter", + # "Nimp_prof.wped", + # "Nimp_prof.peaking", "Te_prof.y0", - "Te_prof.wped", - "Te_prof.wcenter", - "Te_prof.peaking", + # "Te_prof.wped", + # "Te_prof.wcenter", + # "Te_prof.peaking", "Ti_prof.y0", - "Ti_prof.wped", - "Ti_prof.wcenter", - "Ti_prof.peaking", + # "Ti_prof.wped", + # "Ti_prof.wcenter", + # "Ti_prof.peaking", ] OPTIMISED_QUANTITY = ["xrcs.spectra", "cxff_pi.ti", "efit.wp", "smmh1.ne"] @@ -102,6 +104,7 @@ class BayesWorkflow: def __init__( self, pulse=None, + pulse_to_write = 23000101, result_path="./results/example/", nwalkers=50, tstart=0.02, @@ -110,13 +113,17 @@ def __init__( tsample=0.06, iterations=100, burn_in_fraction=0, + + diagnostics=None, center_mass_sampling=True, Ti_ref=True, - diagnostics=None, profiles=None, phantom=True, + mds_write=False, + save_plots=True, ): self.pulse = pulse + self.pulse_to_write = pulse_to_write self.tstart = tstart self.tend = tend self.dt = dt @@ -131,6 +138,8 @@ def __init__( self.diagnostics = diagnostics self.phantom = phantom self.profiles = profiles + self.mds_write = mds_write + self.save_plots = save_plots self.plasma = Plasma( tstart=tstart, @@ -416,7 +425,7 @@ def __call__(self, *args, **kwargs): prior_samples = self.bayes_run.sample_from_priors( OPTIMISED_PARAMS, size=int(1e4) ) - result = { + self.results = { "blobs": blob_dict, "diag_data": self.flat_data, "samples": samples, @@ -424,10 +433,44 @@ def __call__(self, *args, **kwargs): "param_names": OPTIMISED_PARAMS, "phantom_profiles": self.profiles, "autocorr": autocorr, + "acceptance_fraction": self.sampler.acceptance_fraction.sum(), + } + + self.nested_results = { + "DIAG_DATA":{"efit":{"wp":1}}, + # self.ST40_data.binned_data["efit"]["wp"].values[4,] + # "MODEL_DATA": {}, + "METADATA": { + "DIAG_RUNS": "EMPTY", + "EQUIL": self.ST40_data.equilibrium.__str__(), + "IMP1": self.plasma.impurities[0], + "MAIN_ION": self.plasma.main_ion, + }, + "OPTIMISATION": { + "AUTO_CORR":autocorr, + "BURN_FRAC":self.burn_in_fraction, + "ITER": self.iterations, + "NWALKERS": self.nwalkers, + "PARAM_NAMES": OPTIMISED_PARAMS, + "POST_SAMPLE": samples, + "PRIOR_SAMPLE": prior_samples + }, + # "PHANTOMS": self.profiles, + # "PROFILES": {}, + # "TIME": self.plasma.t, + # "TIME_OPT": self.plasma.time_to_calculate + } - self.acceptance_fraction = self.sampler.acceptance_fraction.sum() - print(self.acceptance_fraction) - plot_bayes_result(result, figheader=self.result_path, filetype=".png") + + # self.acceptance_fraction = self.sampler.acceptance_fraction.sum() + print(self.results["acceptance_fraction"]) + + Path(self.result_path).mkdir(parents=True, exist_ok=True) + with open(self.result_path + "results.pkl", "wb") as handle: + pickle.dump(self.results, handle) + + if self.save_plots: + plot_bayes_result(self.results, figheader=self.result_path, filetype=".png") def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): @@ -451,12 +494,20 @@ def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): return autocorr + + if __name__ == "__main__": + + mds_write=False + pulse_to_write = 23000101 + if mds_write: + node_structure = create_nodes(pulse_to_write=pulse_to_write, diagnostic_quantities=OPTIMISED_QUANTITY) + run = BayesWorkflow( pulse=10009, - result_path="./results/10009_60ms_long/", - iterations=500, - nwalkers=200, + result_path="./results/test/", + iterations=20, + nwalkers=50, burn_in_fraction=0.10, dt=0.005, tsample=0.060, @@ -464,5 +515,10 @@ def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): phantom=False, center_mass_sampling=True, Ti_ref=True, + mds_write=mds_write, + save_plots=True ) run() + + if mds_write: + write_nodes(pulse_to_write, node_structure, run.nested_results) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 69ed7a46..3a76d9a3 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -21,8 +21,6 @@ def plot_profile( color="blue", ): set_plot_rcparams("profiles") - # if not plt.get_fignums(): # If no figure is open - # plt.figure(figsize=(8, 6)) if blobkey == "electron_temperature": legkey = "Te" @@ -45,26 +43,27 @@ def plot_profile( label=f"{legkey}, 68% Confidence", zorder=3, color=color, - alpha=0.8, - ) - plt.fill_between( - profile.rho_poloidal, - profile.quantile(0.025, dim="index"), - profile.quantile(0.975, dim="index"), - label=f"{legkey}, 95% Confidence", - zorder=2, - color="grey", - alpha=0.6, - ) - plt.fill_between( - profile.rho_poloidal, - profile.quantile(0.00, dim="index"), - profile.quantile(1.00, dim="index"), - label=f"{legkey}, Max-Min", - zorder=1, - color="lightgrey", - alpha=0.6, + alpha=0.9, ) + if legkey != "Nfast": + plt.fill_between( + profile.rho_poloidal, + profile.quantile(0.025, dim="index"), + profile.quantile(0.975, dim="index"), + label=f"{legkey}, 95% Confidence", + zorder=2, + color="grey", + alpha=0.4, + ) + plt.fill_between( + profile.rho_poloidal, + profile.quantile(0.00, dim="index"), + profile.quantile(1.00, dim="index"), + label=f"{legkey}, Max-Min", + zorder=1, + color="lightgrey", + alpha=0.2, + ) if phantom_profile is not None: if "element" in phantom_profile[blobkey].dims: @@ -237,6 +236,7 @@ def plot_bayes_result( filetype=".png", **kwargs, ): + diag_data = results["diag_data"] blobs = results["blobs"] samples = results["samples"] @@ -245,10 +245,6 @@ def plot_bayes_result( phantom_profiles = results["phantom_profiles"] autocorr = results["autocorr"] - Path(figheader).mkdir(parents=True, exist_ok=True) - with open(figheader + "results.pkl", "wb") as handle: - pickle.dump(results, handle) - plt.figure() plt.plot( np.arange(0, autocorr.__len__())[np.isfinite(autocorr)], @@ -367,6 +363,7 @@ def plot_bayes_result( color="red", linestyle="dotted", ) + key = "electron_density" plot_profile( blobs[key], @@ -374,26 +371,17 @@ def plot_bayes_result( figheader=figheader, filetype=filetype, phantom_profile=phantom_profiles, + color="blue", + sharefig=True ) - key = "impurity_density" - for elem in blobs[key].element.values: - plot_profile( - blobs[key].sel(element=elem), - key, - figheader=figheader, - filename=f"{elem} density", - filetype=filetype, - phantom_profile=phantom_profiles, - color="red", - ) key = "ion_density" plot_profile( blobs[key].sel(element="h"), key, figheader=figheader, - filename=f"h density", filetype=filetype, phantom_profile=phantom_profiles, + sharefig=True, color="red", ) key = "fast_density" @@ -401,10 +389,24 @@ def plot_bayes_result( blobs[key], key, figheader=figheader, + filename="density", filetype=filetype, phantom_profile=phantom_profiles, - color="red", + color="green", ) + + key = "impurity_density" + for elem in blobs[key].element.values: + plot_profile( + blobs[key].sel(element=elem), + key, + figheader=figheader, + filename=f"{elem} density", + filetype=filetype, + phantom_profile=phantom_profiles, + color="red", + ) + key = "neutral_density" plot_profile( blobs[key], @@ -425,7 +427,7 @@ def plot_bayes_result( if __name__ == "__main__": - filehead = "./results/10009_60ms_short/" + filehead = "./results/10009_withoutCM/" with open(filehead + "results.pkl", "rb") as handle: results = pickle.load(handle) plot_bayes_result(results, filehead, filetype=".png") From 533a44a526ff533a1fd03f9d4c0a79568e39cb0e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 22 Jun 2023 16:05:31 +0100 Subject: [PATCH 053/288] multiplot formatting --- indica/workflows/bayes_plots.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 3a76d9a3..403a5cf2 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -170,7 +170,7 @@ def violinplot( figheader="./results/test/", **kwargs, ): - set_plot_rcparams() + set_plot_rcparams("multi") fig, axs = plt.subplots( 1, 1, @@ -188,7 +188,8 @@ def violinplot( violin["bodies"][0].set_edgecolor("black") axs.set_xlabel(key) top = axs.get_ylim()[1] - axs.set_ylim(bottom=0, top=top * 1.1) + bot = axs.get_ylim()[0] + axs.set_ylim( top=top * 1.1, bottom=bot * 0.9) axs.set_ylabel(f"{ylabel}") y = diag_data[key].sel(t=data[key].t).values axs.errorbar( @@ -316,6 +317,8 @@ def plot_bayes_result( figheader=figheader, ylabel="Temperature [eV]", ) + + set_plot_rcparams("multi") key = "xrcs.spectra" if key in blobs.keys(): _plot_1d( @@ -427,7 +430,7 @@ def plot_bayes_result( if __name__ == "__main__": - filehead = "./results/10009_withoutCM/" + filehead = "./results/10009_60ms_long/" with open(filehead + "results.pkl", "rb") as handle: results = pickle.load(handle) plot_bayes_result(results, filehead, filetype=".png") From 751ca8139f77e0946be172dcecb09600cdef046e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 27 Jun 2023 10:11:16 +0100 Subject: [PATCH 054/288] generalised writing kinetic profiles to blobs --- indica/bayesmodels.py | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 4ef46d63..d6802160 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -6,10 +6,13 @@ from scipy.stats import uniform np.seterr(all="ignore") - warnings.simplefilter("ignore", category=FutureWarning) +PROFILES = ["electron_temperature", "electron_density", "ion_temperature", + "impurity_density", "fast_density", "neutral_density"] + + def gaussian(x, mean, sigma): return 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-1 / 2 * ((x - mean) / sigma) ** 2) @@ -182,7 +185,7 @@ def ln_posterior(self, parameters: dict, **kwargs): """ ln_prior = self._ln_prior(parameters) - if ln_prior == -np.inf: # Don't call model if outside priors + if ln_prior == -np.inf: # Don't call models if outside priors return -np.inf, {} self.plasma.update_profiles(parameters) @@ -190,29 +193,12 @@ def ln_posterior(self, parameters: dict, **kwargs): ln_likelihood = self._ln_likelihood() # compare results to data ln_posterior = ln_likelihood + ln_prior - kin_profs = { - "electron_density": self.plasma.electron_density.sel( - t=self.plasma.time_to_calculate - ), - "electron_temperature": self.plasma.electron_temperature.sel( - t=self.plasma.time_to_calculate - ), - "ion_temperature": self.plasma.ion_temperature.sel( - t=self.plasma.time_to_calculate - ), - "impurity_density": self.plasma.impurity_density.sel( - t=self.plasma.time_to_calculate - ), - "ion_density": self.plasma.ion_density.sel( - t=self.plasma.time_to_calculate - ), - "fast_density": self.plasma.fast_density.sel( - t=self.plasma.time_to_calculate - ), - "neutral_density": self.plasma.neutral_density.sel( - t=self.plasma.time_to_calculate - ), - - } + kin_profs = {} + for profile_key in PROFILES: + if hasattr(self.plasma, profile_key): + kin_profs[profile_key] = getattr(self.plasma, profile_key).sel(t=self.plasma.time_to_calculate) + else: + raise ValueError(f"plasma does not have attribute {profile_key}") + blob = deepcopy({**self.bckc, **kin_profs}) return ln_posterior, blob From 59c8316053de2d232e41c63174b6939b6429c355 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 27 Jun 2023 11:25:43 +0100 Subject: [PATCH 055/288] function autocorr plot added --- indica/workflows/bayes_plots.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 403a5cf2..ea943eae 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -231,6 +231,23 @@ def histograms(data, diag_data, filename): plt.close() + +def plot_autocorr(autocorr, figheader, filetype=".png"): + + plt.figure() + plt.plot( + np.arange(0, autocorr.__len__())[np.isfinite(autocorr)], + autocorr[np.isfinite(autocorr)], + label="average auto-correlation time", + ) + + plt.legend() + plt.xlabel("iterations") + plt.ylabel("auto-correlation time (iterations)") + plt.savefig(figheader + "average_tau" + filetype) + plt.close() + + def plot_bayes_result( results, figheader="./results/test/", @@ -246,17 +263,7 @@ def plot_bayes_result( phantom_profiles = results["phantom_profiles"] autocorr = results["autocorr"] - plt.figure() - plt.plot( - np.arange(0, autocorr.__len__())[np.isfinite(autocorr)], - autocorr[np.isfinite(autocorr)], - label="average tau", - ) - plt.legend() - plt.xlabel("iterations") - plt.ylabel("auto-correlation time (iterations)") - plt.savefig(figheader + "average_tau" + filetype) - plt.close() + plot_autocorr(autocorr, figheader, filetype=filetype) if "cxff_pi.ti" in blobs.keys(): blobs["cxff_pi.ti0"] = blobs["cxff_pi.ti"].sel( @@ -430,7 +437,7 @@ def plot_bayes_result( if __name__ == "__main__": - filehead = "./results/10009_60ms_long/" + filehead = "./results/test/" with open(filehead + "results.pkl", "rb") as handle: results = pickle.load(handle) plot_bayes_result(results, filehead, filetype=".png") From b2dee2d22d76d146f4f2c5836e7659f74e766f42 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 27 Jun 2023 11:26:12 +0100 Subject: [PATCH 056/288] ion density added to kinetic profiles saved --- indica/bayesmodels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index d6802160..fc791837 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -9,7 +9,7 @@ warnings.simplefilter("ignore", category=FutureWarning) -PROFILES = ["electron_temperature", "electron_density", "ion_temperature", +PROFILES = ["electron_temperature", "electron_density", "ion_temperature", "ion_density", "impurity_density", "fast_density", "neutral_density"] From 8eee24ebfd9516e55f2bf914764e69c796c7fd8d Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 27 Jun 2023 16:04:21 +0100 Subject: [PATCH 057/288] inital commit of abstract workflow and example of usage --- indica/workflows/abstract_bayes_workflow.py | 47 +++ indica/workflows/test_bayes_workflow.py | 336 ++++++++++++++++++++ 2 files changed, 383 insertions(+) create mode 100644 indica/workflows/abstract_bayes_workflow.py create mode 100644 indica/workflows/test_bayes_workflow.py diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py new file mode 100644 index 00000000..95d8d6d0 --- /dev/null +++ b/indica/workflows/abstract_bayes_workflow.py @@ -0,0 +1,47 @@ +from abc import ABC, abstractmethod + + +class BayesWorkflow(ABC): + @abstractmethod + def setup_plasma(self): + """ + Contains all methods and settings for plasma object to be used in optimisation + """ + self.plasma = None + + @abstractmethod + def read_data(self, diagnostics: list): + """ + Reads data from server + + Returns + + dictionary of data + """ + self.data = {} + + def setup_opt_data(self, phantom: bool = False): + """ + Prepare the data in necessary format for optimiser + """ + self.opt_data = {} + + @abstractmethod + def setup_models(self, diagnostics: list): + """ + Initialise models normally requires data to be read so transforms can be set + + """ + self.models = {} + + @abstractmethod + def setup_optimiser(self): + """ + Initialise and provide settings for optimiser + """ + self.bayesopt = None + + @abstractmethod + def __call__(self, *args, **kwargs): + return {} + diff --git a/indica/workflows/test_bayes_workflow.py b/indica/workflows/test_bayes_workflow.py new file mode 100644 index 00000000..f7d4206d --- /dev/null +++ b/indica/workflows/test_bayes_workflow.py @@ -0,0 +1,336 @@ +import emcee +import numpy as np +import flatdict +from scipy.stats import loguniform + +from indica.readers.read_st40 import ReadST40 +from indica.bayesmodels import BayesModels, get_uniform +from indica.models.interferometry import Interferometry +from indica.models.helike_spectroscopy import Helike_spectroscopy +from indica.models.charge_exchange import ChargeExchange +from indica.models.equilibrium_reconstruction import EquilibriumReconstruction +from indica.models.plasma import Plasma +from indica.converters.line_of_sight import LineOfSightTransform + +from abstract_bayes_workflow import BayesWorkflow + +# global configurations +DEFAULT_PHANTOM_PARAMS = { + "Ne_prof.y0": 5e19, + "Ne_prof.wcenter": 0.4, + "Ne_prof.peaking": 2, + "Ne_prof.y1": 2e18, + "Ne_prof.yend": 1e18, + "Ne_prof.wped": 2, + "Nimp_prof.y0": 3e16, + "Nimp_prof.y1": 0.5e16, + "Nimp_prof.wcenter": 0.4, + "Nimp_prof.wped": 6, + "Nimp_prof.peaking": 2, + "Te_prof.y0": 3000, + "Te_prof.y1": 50, + "Te_prof.wcenter": 0.4, + "Te_prof.wped": 3, + "Te_prof.peaking": 2, + "Ti_prof.y0": 6000, + "Ti_prof.y1": 50, + "Ti_prof.wcenter": 0.4, + "Ti_prof.wped": 3, + "Ti_prof.peaking": 2, +} + +DEFAULT_PRIORS = { + "Ne_prof.y0": get_uniform(2e19, 4e20), + "Ne_prof.y1": get_uniform(1e18, 1e19), + "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), + "Ne_prof.wped": get_uniform(1, 6), + "Ne_prof.wcenter": get_uniform(0.1, 0.8), + "Ne_prof.peaking": get_uniform(1, 6), + "ar_conc": loguniform(0.0001, 0.01), + "Nimp_prof.y0": get_uniform(1e16, 1e18), + "Nimp_prof.y1": get_uniform(1e15, 2e16), + "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( + (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 + ), + "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), + "Nimp_prof.wped": get_uniform(1, 6), + "Nimp_prof.wcenter": get_uniform(0.1, 0.8), + "Nimp_prof.peaking": get_uniform(1, 20), + "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where( + (x1 > x2), 1, 0 + ), # impurity always more peaked + "Te_prof.y0": get_uniform(1000, 5000), + "Te_prof.wped": get_uniform(1, 6), + "Te_prof.wcenter": get_uniform(0.1, 0.6), + "Te_prof.peaking": get_uniform(1, 6), + "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode + "Ti_prof.y0": get_uniform(3000, 10000), + "Ti_prof.wped": get_uniform(1, 6), + "Ti_prof.wcenter": get_uniform(0.1, 0.6), + "Ti_prof.peaking": get_uniform(1, 20), +} + +OPTIMISED_PARAMS = [ + # "Ne_prof.y1", + "Ne_prof.y0", + "Ne_prof.peaking", + "Ne_prof.wcenter", + "Ne_prof.wped", + # "ar_conc", + # "Nimp_prof.y1", + "Nimp_prof.y0", + # "Nimp_prof.wcenter", + # "Nimp_prof.wped", + # "Nimp_prof.peaking", + "Te_prof.y0", + # "Te_prof.wped", + # "Te_prof.wcenter", + # "Te_prof.peaking", + "Ti_prof.y0", + # "Ti_prof.wped", + # "Ti_prof.wcenter", + # "Ti_prof.peaking", +] + +OPTIMISED_QUANTITY = ["cxff_pi.ti", "efit.wp", "smmh1.ne"] + + +class TestWorkflow(BayesWorkflow): + def __init__( + self, + pulse=None, + nwalkers=50, + tstart=0.02, + tend=0.10, + dt=0.01, + tsample=0.06, + iterations=100, + burn_in_fraction=0, + + phantom=False, + diagnostics=None, + ): + self.pulse = pulse + self.tstart = tstart + self.tend = tend + self.dt = dt + self.tsample = tsample + self.nwalkers = nwalkers + self.iterations = iterations + self.burn_in_fraction = burn_in_fraction + + self.phantom = phantom + self.diagnostics = diagnostics + + self.setup_plasma() + self.read_data(diagnostics) + self.setup_opt_data(phantom=self.phantom) + self.setup_models(diagnostics) + self.setup_optimiser() + + def setup_plasma(self): + self.plasma = Plasma( + tstart=self.tstart, + tend=self.tend, + dt=self.dt, + main_ion="h", + impurities=("ar", "c"), + impurity_concentration=( + 0.001, + 0.02, + ), + full_run=False, + n_rad=20, + ) + self.plasma.Ti_ref = True + + self.plasma.time_to_calculate = self.plasma.t[ + np.abs(self.tsample - self.plasma.t).argmin() + ] + self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) + self.plasma.build_atomic_data(calc_power_loss=False) + + def read_data(self, diagnostics: list): + self.reader = ReadST40( + self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt + ) + self.reader(diagnostics) + self.plasma.set_equilibrium(self.reader.equilibrium) + self.data = self.reader.binned_data + + def setup_opt_data(self, phantom=False): + if phantom: + self._phantom_data() + else: + self._exp_data() + + def setup_models(self, diagnostics: list): + self.models = {} + for diag in self.diagnostics: + if diag == "smmh1": + # los_transform = self.data["smmh1"]["ne"].transform + machine_dims = self.plasma.machine_dimensions + origin = np.array([[-0.38063365, 0.91893092, 0.01]]) + # end = np.array([[0, 0, 0.01]]) + direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) + los_transform = LineOfSightTransform( + origin[:, 0], + origin[:, 1], + origin[:, 2], + direction[:, 0], + direction[:, 1], + direction[:, 2], + name="", + machine_dimensions=machine_dims, + passes=2, + ) + los_transform.set_equilibrium(self.plasma.equilibrium) + model = Interferometry(name=diag) + model.set_los_transform(los_transform) + model.plasma = self.plasma + + elif diag == "xrcs": + los_transform = self.data["xrcs"]["te_kw"].transform + model = Helike_spectroscopy( + name="xrcs", + window_masks=[slice(0.394, 0.396)], + window_vector=self.data[diag][ + "spectra" + ].wavelength.values + * 0.1, + ) + model.set_los_transform(los_transform) + model.plasma = self.plasma + + elif diag == "efit": + model = EquilibriumReconstruction(name="efit") + model.plasma = self.plasma + + elif diag == "cxff_pi": + transform = self.data[diag]["ti"].transform + transform.set_equilibrium(self.plasma.equilibrium) + model = ChargeExchange(name=diag, element="ar") + model.set_transect_transform(transform) + model.plasma = self.plasma + else: + raise ValueError(f"{diag} not found in setup_models") + + self.models[diag] = model + + + def setup_optimiser(self, ): + + self.bayesopt = BayesModels( + plasma=self.plasma, + data=self.opt_data, + diagnostic_models=[*self.models.values()], + quant_to_optimise=OPTIMISED_QUANTITY, + priors=DEFAULT_PRIORS, + ) + + ndim = len(OPTIMISED_PARAMS) + + self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] + self.sampler = emcee.EnsembleSampler( + self.nwalkers, + ndim, + log_prob_fn=self.bayesopt.ln_posterior, + parameter_names=OPTIMISED_PARAMS, + moves=self.move, + ) + + self.start_points = self.bayesopt.sample_from_priors( + OPTIMISED_PARAMS, size=self.nwalkers + ) + + + def _phantom_data(self, noise=False, noise_factor=0.1): + self.opt_data = {} + if "smmh1" in self.diagnostics: + self.opt_data["smmh1.ne"] = ( + self.models["smmh1"]() + .pop("ne") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + ) + if "xrcs" in self.diagnostics: + self.opt_data["xrcs.spectra"] = ( + self.models["xrcs"]() + .pop("spectra") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + ) + self.opt_data["xrcs.background"] = None + if "cxff_pi" in self.diagnostics: + cxrs_data = ( + self.models["cxff_pi"]() + .pop("ti") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + ) + self.opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) + if "efit" in self.diagnostics: + self.opt_data["efit.wp"] = ( + self.models["efit"]() + .pop("wp") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + ) + + if noise: + self.opt_data["smmh1.ne"] = self.opt_data["smmh1.ne"] + self.opt_data[ + "smmh1.ne" + ].max().values * np.random.normal(0, noise_factor, None) + self.opt_data["xrcs.spectra"] = self.opt_data[ + "xrcs.spectra" + ] + np.random.normal( + 0, + np.sqrt(self.opt_data["xrcs.spectra"].values[0,]), + self.opt_data["xrcs.spectra"].shape[1], + ) + self.opt_data["cxff_pi.ti"] = self.opt_data[ + "cxff_pi.ti" + ] + self.opt_data["cxff_pi.ti"].max().values * np.random.normal( + 0, noise_factor, self.opt_data["cxff_pi.ti"].shape[1] + ) + self.opt_data["efit.wp"] = self.opt_data["efit.wp"] + self.opt_data[ + "efit.wp" + ].max().values * np.random.normal(0, noise_factor, None) + + def _exp_data(self): + self.opt_data = flatdict.FlatDict(self.data, ".") + if "xrcs" in self.diagnostics: + self.opt_data["xrcs.spectra"]["wavelength"] = ( + self.opt_data["xrcs.spectra"].wavelength * 0.1 + ) + background = self.opt_data["xrcs.spectra"].where( + (self.opt_data["xrcs.spectra"].wavelength < 0.392) + & (self.opt_data["xrcs.spectra"].wavelength > 0.388), + drop=True, + ) + self.opt_data["xrcs.background"] = background.mean(dim="wavelength") + self.opt_data["xrcs.spectra"]["error"] = np.sqrt( + self.opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 + ) + + if "cxff_pi" in self.diagnostics: + self.opt_data["cxff_pi"]["ti"] = self.opt_data["cxff_pi"]["ti"].where( + self.opt_data["cxff_pi"]["ti"].channel == 2, drop=True + ) + + def __call__(self, *args, **kwargs): + self.sampler.run_mcmc(self.start_points, self.iterations, progress=True) + blobs = self.sampler.get_blobs( + discard=int(self.iterations * self.burn_in_fraction), flat=True + ) + return blobs + + +if __name__ == "__main__": + run = TestWorkflow( + pulse=10009, + dt=0.005, + tsample=0.060, + diagnostics=["efit", "smmh1", "cxff_pi"], + + iterations=10, + nwalkers=20, + burn_in_fraction=0.10, + ) + run() From 408b619627d4e65a7897e3504759a1ad152f559e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 27 Jun 2023 16:04:53 +0100 Subject: [PATCH 058/288] doc strings --- indica/workflows/abstract_bayes_workflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 95d8d6d0..caa14bc4 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -16,20 +16,20 @@ def read_data(self, diagnostics: list): Returns - dictionary of data + nested dictionary of data """ self.data = {} def setup_opt_data(self, phantom: bool = False): """ - Prepare the data in necessary format for optimiser + Prepare the data in necessary format for optimiser i.e. flat dictionary """ self.opt_data = {} @abstractmethod def setup_models(self, diagnostics: list): """ - Initialise models normally requires data to be read so transforms can be set + Initialising models normally requires data to be read so transforms can be set """ self.models = {} From de237dd3c68976e740c834e79978aa7457621c16 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 27 Jun 2023 16:06:27 +0100 Subject: [PATCH 059/288] renaming workflow --- .../{test_bayes_workflow.py => bayes_workflow_example.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename indica/workflows/{test_bayes_workflow.py => bayes_workflow_example.py} (100%) diff --git a/indica/workflows/test_bayes_workflow.py b/indica/workflows/bayes_workflow_example.py similarity index 100% rename from indica/workflows/test_bayes_workflow.py rename to indica/workflows/bayes_workflow_example.py From 06c07462aea64efdfbb537ccc8629cd772ace81f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 30 Jun 2023 13:38:43 +0100 Subject: [PATCH 060/288] Ti_ref as default --- indica/models/plasma.py | 11 ++----- ..._dev_workflow.py => bayes_workflow_dev.py} | 33 ++++++++++--------- 2 files changed, 19 insertions(+), 25 deletions(-) rename indica/workflows/{bayes_dev_workflow.py => bayes_workflow_dev.py} (96%) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 1f57439c..fcf79388 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -462,16 +462,9 @@ def assign_profiles(self, impurity_to_profile=None): self.electron_density.loc[dict(t=t)] = self.Ne_prof() self.calc_impurity_density(self.impurities) self.electron_temperature.loc[dict(t=t)] = self.Te_prof() - if self.Ti_ref: - self.ion_temperature.loc[dict(t=t)] = self.Ti_prof(y0_ref=self.Te_prof.y0) - else: - self.ion_temperature.loc[dict(t=t)] = self.Ti_prof() + self.ion_temperature.loc[dict(t=t)] = self.Ti_prof(y0_ref=self.Te_prof.y0) # Temp default Ti_ref self.toroidal_rotation.loc[dict(t=t)] = self.Vrot_prof() - self.neutral_density.loc[ - dict( - t=t, - ) - ] = self.Nh_prof() + self.neutral_density.loc[dict(t=t)] = self.Nh_prof() if ( impurity_to_profile is not None ): # overwrite impurity profile with non-Ne dependent profile diff --git a/indica/workflows/bayes_dev_workflow.py b/indica/workflows/bayes_workflow_dev.py similarity index 96% rename from indica/workflows/bayes_dev_workflow.py rename to indica/workflows/bayes_workflow_dev.py index d6e72780..5961bb5f 100644 --- a/indica/workflows/bayes_dev_workflow.py +++ b/indica/workflows/bayes_workflow_dev.py @@ -97,7 +97,7 @@ # "Ti_prof.peaking", ] -OPTIMISED_QUANTITY = ["xrcs.spectra", "cxff_pi.ti", "efit.wp", "smmh1.ne"] +OPTIMISED_QUANTITY = ["cxff_pi.ti", "efit.wp", "smmh1.ne"] class BayesWorkflow: @@ -161,9 +161,10 @@ def __init__( ] self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) self.plasma.build_atomic_data(calc_power_loss=False) - self.init_fast_particles() + self.read_st40(diagnostics) + self.init_models() if self.phantom: @@ -188,12 +189,12 @@ def __init__( log_prob_fn=self.bayes_run.ln_posterior, parameter_names=OPTIMISED_PARAMS, moves=self.move, - kwargs={ - "moment_analysis": False, - "calc_spectra": True, - "minimum_lines": False, - "background": self.flat_data["xrcs.background"], - }, + # kwargs={ + # "moment_analysis": False, + # "calc_spectra": True, + # "minimum_lines": False, + # "background": self.flat_data["xrcs.background"], + # }, ) if not self.center_mass_sampling: @@ -407,7 +408,7 @@ def exp_data(self): def __call__(self, *args, **kwargs): autocorr = sample_with_autocorr( - self.sampler, self.start_points, iterations=self.iterations, auto_sample=5 + self.sampler, self.start_points, self.iterations, auto_sample=10 ) blobs = self.sampler.get_blobs( discard=int(self.iterations * self.burn_in_fraction), flat=True @@ -473,7 +474,7 @@ def __call__(self, *args, **kwargs): plot_bayes_result(self.results, figheader=self.result_path, filetype=".png") -def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): +def sample_with_autocorr(sampler, start_points, iterations, auto_sample=5): autocorr = np.ones((iterations,)) * np.nan old_tau = np.inf for sample in sampler.sample( @@ -483,14 +484,14 @@ def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): ): if sampler.iteration % auto_sample: continue - new_tau = sampler.get_autocorr_time(tol=0) - autocorr[sampler.iteration - 1] = np.mean(new_tau) + new_tau = np.mean(sampler.get_autocorr_time(tol=0)) + autocorr[sampler.iteration - 1] = new_tau converged = np.all(new_tau * 50 < sampler.iteration) converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) if converged: break old_tau = new_tau - autocorr = autocorr[: sampler.iteration] + autocorr = autocorr[: sampler.iteration, ] return autocorr @@ -507,13 +508,13 @@ def sample_with_autocorr(sampler, start_points, iterations=10, auto_sample=5): pulse=10009, result_path="./results/test/", iterations=20, - nwalkers=50, + nwalkers=20, burn_in_fraction=0.10, dt=0.005, tsample=0.060, - diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], + diagnostics=["efit", "smmh1", "cxff_pi"], phantom=False, - center_mass_sampling=True, + center_mass_sampling=False, Ti_ref=True, mds_write=mds_write, save_plots=True From 5435769855061ccc964a735793840d18f1fe12a1 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 30 Jun 2023 13:39:30 +0100 Subject: [PATCH 061/288] abstract methods added --- indica/workflows/abstract_bayes_workflow.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index caa14bc4..d2bb2f69 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -20,6 +20,8 @@ def read_data(self, diagnostics: list): """ self.data = {} + + @abstractmethod def setup_opt_data(self, phantom: bool = False): """ Prepare the data in necessary format for optimiser i.e. flat dictionary From ee3a62e1af7ef4a8cafcf4897e6d2907d89c4ccd Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 14:08:00 +0100 Subject: [PATCH 062/288] Changed to new MDSPlus structure --- indica/workflows/bayes_plots.py | 186 +++++++++++++++++--------------- 1 file changed, 97 insertions(+), 89 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index ea943eae..9d781436 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -1,9 +1,10 @@ -from pathlib import Path +import os import corner import matplotlib.pyplot as plt import numpy as np import pickle +import flatdict from indica.utilities import set_plot_rcparams, set_axis_sci @@ -22,35 +23,21 @@ def plot_profile( ): set_plot_rcparams("profiles") - if blobkey == "electron_temperature": - legkey = "Te" - elif blobkey == "ion_temperature": - legkey = "Ti" - elif blobkey == "ion_density": - legkey = "Ni" - elif blobkey == "electron_density": - legkey = "Ne" - elif blobkey == "impurity_density": - legkey = "Nimp" - elif blobkey == "fast_density": - legkey = "Nfast" - elif blobkey == "neutral_density": - legkey = "Nneut" plt.fill_between( profile.rho_poloidal, profile.quantile(0.16, dim="index"), profile.quantile(0.84, dim="index"), - label=f"{legkey}, 68% Confidence", + label=f"{blobkey}, 68% Confidence", zorder=3, color=color, alpha=0.9, ) - if legkey != "Nfast": + if blobkey != "NFAST": plt.fill_between( profile.rho_poloidal, profile.quantile(0.025, dim="index"), profile.quantile(0.975, dim="index"), - label=f"{legkey}, 95% Confidence", + label=f"{blobkey}, 95% Confidence", zorder=2, color="grey", alpha=0.4, @@ -59,19 +46,19 @@ def plot_profile( profile.rho_poloidal, profile.quantile(0.00, dim="index"), profile.quantile(1.00, dim="index"), - label=f"{legkey}, Max-Min", + label=f"{blobkey}, Max-Min", zorder=1, color="lightgrey", alpha=0.2, ) - if phantom_profile is not None: + if phantom_profile["FLAG"]: if "element" in phantom_profile[blobkey].dims: phantom = phantom_profile[blobkey].sel(element="ar") else: phantom = phantom_profile[blobkey] phantom.plot( - label=f"{legkey}, phantom profile", + label=f"{blobkey}, phantom profile", linestyle=linestyle, color="black", zorder=4, @@ -232,16 +219,17 @@ def histograms(data, diag_data, filename): -def plot_autocorr(autocorr, figheader, filetype=".png"): +def plot_autocorr(autocorr, param_names, figheader, filetype=".png"): plt.figure() + + x_data = np.ones(shape=(autocorr.shape)) * np.arange(0, autocorr[:,0].__len__())[:,None] plt.plot( - np.arange(0, autocorr.__len__())[np.isfinite(autocorr)], - autocorr[np.isfinite(autocorr)], - label="average auto-correlation time", + np.where(np.isfinite(autocorr), x_data, np.nan), + autocorr, + "x" ) - - plt.legend() + plt.legend(param_names) plt.xlabel("iterations") plt.ylabel("auto-correlation time (iterations)") plt.savefig(figheader + "average_tau" + filetype) @@ -255,28 +243,38 @@ def plot_bayes_result( **kwargs, ): - diag_data = results["diag_data"] - blobs = results["blobs"] - samples = results["samples"] - prior_samples = results["prior_samples"] - param_names = results["param_names"] - phantom_profiles = results["phantom_profiles"] - autocorr = results["autocorr"] - - plot_autocorr(autocorr, figheader, filetype=filetype) - - if "cxff_pi.ti" in blobs.keys(): - blobs["cxff_pi.ti0"] = blobs["cxff_pi.ti"].sel( - channel=diag_data["cxff_pi.ti"].channel + # delete all but pickle in directory + for root, dirs, files in os.walk(figheader): + for f in files: + if f.endswith(".pkl"): + continue + else: + print(f"Deleting {os.path.join(root, f)}") + os.remove(os.path.join(root, f)) + + diag_data = flatdict.FlatDict(results["DIAG_DATA"], ".") + model_data = flatdict.FlatDict(results["MODEL_DATA"], ".") + profiles = flatdict.FlatDict(results["PROFILE_STAT"], ".") + post_sample = results["OPTIMISATION"]["POST_SAMPLE"] + prior_sample = results["OPTIMISATION"]["PRIOR_SAMPLE"] + autocorr = results["OPTIMISATION"]["AUTOCORR"] + param_names = results["INPUT"]["PARAM_NAMES"] + phantom_profiles = flatdict.FlatDict(results["PHANTOMS"], ".") + + plot_autocorr(autocorr, param_names, figheader, filetype=filetype) + + if "CXFF_PI.TI" in model_data.keys(): + model_data["CXFF_PI.TI0"] = model_data["CXFF_PI.TI"].sel( + channel=diag_data["CXFF_PI.TI"].channel ) - diag_data["cxff_pi.ti0"] = diag_data["cxff_pi.ti"].sel( - channel=diag_data["cxff_pi.ti"].channel + diag_data["CXFF_PI.TI0"] = diag_data["CXFF_PI.TI"].sel( + channel=diag_data["CXFF_PI.TI"].channel ) - key = "cxff_pi.ti0" - if key in blobs.keys(): + key = "CXFF_PI.TI0" + if key in model_data.keys(): violinplot( - blobs, + model_data, key, diag_data, f"{key.replace('.', '_')}" + filetype, @@ -284,10 +282,10 @@ def plot_bayes_result( ylabel="Temperature [eV]", ) - key = "efit.wp" - if key in blobs.keys(): + key = "EFIT.WP" + if key in model_data.keys(): violinplot( - blobs, + model_data, key, diag_data, f"{key.replace('.', '_')}" + filetype, @@ -295,29 +293,29 @@ def plot_bayes_result( ylabel="Energy [J]", ) key = "smmh1.ne" - if key in blobs.keys(): + if key in model_data.keys(): violinplot( - blobs, + model_data, key, diag_data, f"{key.replace('.', '_')}" + filetype, figheader=figheader, ylabel=r"Line Integrated Density [$m^{-2}$]", ) - key = "xrcs.te_kw" - if key in blobs.keys(): + key = "XRCS.TE_KW" + if key in model_data.keys(): violinplot( - blobs, + model_data, key, diag_data, f"{key.replace('.', '_')}" + filetype, figheader=figheader, ylabel="Temperature [eV]", ) - key = "xrcs.ti_w" - if key in blobs.keys(): + key = "XRCS.TI_W" + if key in model_data.keys(): violinplot( - blobs, + model_data, key, diag_data, f"{key.replace('.', '_')}" + filetype, @@ -326,10 +324,10 @@ def plot_bayes_result( ) set_plot_rcparams("multi") - key = "xrcs.spectra" - if key in blobs.keys(): + key = "XRCS.SPECTRA" + if key in model_data.keys(): _plot_1d( - blobs, + model_data, key, diag_data, f"{key.replace('.', '_')}" + filetype, @@ -339,10 +337,10 @@ def plot_bayes_result( xlim=(0.394, 0.401), figsize=(6, 4), ) - key = "cxff_pi.ti" - if key in blobs.keys(): + key = "CXFF_PI.TI" + if key in model_data.keys(): _plot_1d( - blobs, + model_data, key, diag_data, f"{key.replace('.', '_')}" + filetype, @@ -351,9 +349,9 @@ def plot_bayes_result( xlabel="Channel", ) - key = "electron_temperature" + key = "TE" plot_profile( - blobs[key], + profiles[key], key, figheader=figheader, filetype=filetype, @@ -362,9 +360,9 @@ def plot_bayes_result( color="blue", linestyle="dashdot", ) - key = "ion_temperature" + key = "TI" plot_profile( - blobs[key].sel(element="ar"), + profiles[key].sel(element="ar"), key, figheader=figheader, filename="temperature", @@ -374,9 +372,9 @@ def plot_bayes_result( linestyle="dotted", ) - key = "electron_density" + key = "NE" plot_profile( - blobs[key], + profiles[key], key, figheader=figheader, filetype=filetype, @@ -384,9 +382,9 @@ def plot_bayes_result( color="blue", sharefig=True ) - key = "ion_density" + key = "NI" plot_profile( - blobs[key].sel(element="h"), + profiles[key].sel(element="h"), key, figheader=figheader, filetype=filetype, @@ -394,44 +392,54 @@ def plot_bayes_result( sharefig=True, color="red", ) - key = "fast_density" + key = "NFAST" plot_profile( - blobs[key], + profiles[key], key, figheader=figheader, - filename="density", + filename="densities", filetype=filetype, phantom_profile=phantom_profiles, color="green", ) - key = "impurity_density" - for elem in blobs[key].element.values: - plot_profile( - blobs[key].sel(element=elem), - key, - figheader=figheader, - filename=f"{elem} density", - filetype=filetype, - phantom_profile=phantom_profiles, - color="red", - ) + key = "NIMP1" + plot_profile( + profiles[key], + key, + figheader=figheader, + filename=f"ar density", + filetype=filetype, + phantom_profile=phantom_profiles, + color="red", + ) + + key = "NIMP2" + plot_profile( + profiles[key], + key, + figheader=figheader, + filename=f"c density", + filetype=filetype, + phantom_profile=phantom_profiles, + color="red", + ) - key = "neutral_density" + key = "NNEUTR" plot_profile( - blobs[key], + profiles[key], key, - filename="", + filename="neutral density", figheader=figheader, filetype=filetype, phantom_profile=phantom_profiles, logscale=True, ) - corner.corner(samples, labels=param_names) + corner.corner(post_sample, labels=param_names) plt.savefig(figheader + "posterior" + filetype) - corner.corner(prior_samples, labels=param_names) + corner.corner(prior_sample, labels=param_names) plt.savefig(figheader + "prior" + filetype) plt.close("all") From a8ca5fab7bc4e2a8013bdab68bf35d3cc1dc886e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 14:08:47 +0100 Subject: [PATCH 063/288] Adding default methods --- indica/workflows/abstract_bayes_workflow.py | 134 +++++++++++++++++++- 1 file changed, 133 insertions(+), 1 deletion(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index d2bb2f69..fdf2d5bc 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -1,7 +1,21 @@ from abc import ABC, abstractmethod - +import numpy as np class BayesWorkflow(ABC): + def __init__(self, + phantom, + diagnostics): + + self.phantom = phantom + self.diagnostics = diagnostics + + self.setup_plasma() + self.save_phantom_profiles() + self.read_data(self.diagnostics) + self.setup_opt_data(self.phantom) + self.setup_models(self.diagnostics) + self.setup_optimiser() + @abstractmethod def setup_plasma(self): """ @@ -43,7 +57,125 @@ def setup_optimiser(self): """ self.bayesopt = None + def save_phantom_profiles(self, kinetic_profiles=None): + if kinetic_profiles is None: + kinetic_profiles = ["electron_density", "impurity_density", "electron_temperature", + "ion_temperature", "ion_density", "fast_density", "neutral_density"] + if self.phantoms: + phantom_profiles = {profile_key: getattr(self.plasma, profile_key).sel( + t=self.plasma.time_to_calculate).copy() + for profile_key in kinetic_profiles} + else: + phantom_profiles = {profile_key: getattr(self.plasma, profile_key).sel( + t=self.plasma.time_to_calculate) * 0 + for profile_key in kinetic_profiles} + + self.phantom_profiles = phantom_profiles + + def _build_result_dict(self): + + """ + TODO: xarray to numpy for MDSPlus writing (plotting / dimensions / units have to be figured out) + + + """ + + + result = {} + result["TIME"] = self.plasma.t + result["TIME_OPT"] = self.plasma.time_to_calculate + + result["INPUT"] = { + "BURN_FRAC": self.burn_frac, + "ITER": self.iterations, + "MODEL_KWARGS": "KWARGS", + "OPT_KWARGS": "KWARGS", + "NWALKERS": self.nwalkers, + "PARAM_NAMES": self.param_names, + "PULSE": self.pulse, + } + + result["GLOBAL"] = { + "TI0": 0, + "TE0": 0, + "NE0": 0, + "NI0": 0, + "TI0_ERR": 0, + "TE0_ERR": 0, + "NE0_ERR": 0, + "NI0_ERR": 0, + } + + result["PHANTOMS"] = { + "FLAG": self.phantoms, + "NE": self.phantom_profiles["electron_density"], + "TE": self.phantom_profiles["electron_temperature"], + "TI": self.phantom_profiles["ion_temperature"], + "NI": self.phantom_profiles["ion_density"], + "NNEUTR": self.phantom_profiles["neutral_density"], + "NFAST": self.phantom_profiles["fast_density"], + "NIMP1": self.phantom_profiles["impurity_density"].sel(element="ar"), # TODO: generalise + "NIMP2": self.phantom_profiles["impurity_density"].sel(element="c") + } + + result["PROFILE_STAT"] = { + "SAMPLES": self.samples, + "RHO_POLOIDAL": self.plasma.rho, + "NE": self.blobs["electron_density"], + "NI": self.blobs["ion_density"], + "TE": self.blobs["electron_temperature"], + "TI": self.blobs["ion_temperature"], + "NFAST": self.blobs["fast_density"], + "NNEUTR": self.blobs["neutral_density"], + "NIMP1": self.blobs["impurity_density"].sel(element="ar"), # TODO: generalise + "NIMP2": self.blobs["impurity_density"].sel(element="c") + } + + result["OPTIMISATION"] = { + "OPT_QUANTITY": self.opt_quantity, + "ACCEPT_FRAC": self.accept_frac, + "PRIOR_SAMPLE": self.prior_sample, + "POST_SAMPLE": self.post_sample, + "AUTOCORR": self.autocorr, + } + + quant_list = [item.split(".") for item in self.opt_quantity] + result["MODEL_DATA"] = {diag_name.upper(): + {quantity[1].upper(): self.blobs[f"{quantity[0]}.{quantity[1]}"] + for quantity in quant_list if quantity[0] == diag_name} + for diag_name in self.diagnostics + } + result["MODEL_DATA"]["SAMPLES"] = self.samples + + result["DIAG_DATA"] = {diag_name.upper(): + {quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] + for quantity in quant_list if quantity[0] == diag_name} + for diag_name in self.diagnostics + } + self.result = result + return self.result + @abstractmethod def __call__(self, *args, **kwargs): return {} + +def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sample=5): + autocorr = np.ones(shape=(iterations, n_params)) * np.nan + old_tau = np.inf + for sample in sampler.sample( + start_points, + iterations=iterations, + progress=True, + ): + if sampler.iteration % auto_sample: + continue + new_tau = sampler.get_autocorr_time(tol=0) + autocorr[sampler.iteration - 1, ] = new_tau + converged = np.all(new_tau * 50 < sampler.iteration) + converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) + if converged: + break + old_tau = new_tau + autocorr = autocorr[: sampler.iteration, ] + return autocorr \ No newline at end of file From 6f1e3308ea0ddebe3dbf7c93a467c2ca9a3ac93e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 15:09:48 +0100 Subject: [PATCH 064/288] moved sampling to it's own function in abstract class --- indica/workflows/bayes_workflow_example.py | 84 ++++++++++++++-------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index f7d4206d..4bc2e39a 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -1,7 +1,12 @@ +import pickle + import emcee import numpy as np +import xarray as xr +import pandas as pd import flatdict from scipy.stats import loguniform +from pathlib import Path from indica.readers.read_st40 import ReadST40 from indica.bayesmodels import BayesModels, get_uniform @@ -12,9 +17,11 @@ from indica.models.plasma import Plasma from indica.converters.line_of_sight import LineOfSightTransform -from abstract_bayes_workflow import BayesWorkflow +from abstract_bayes_workflow import BayesWorkflow, sample_with_autocorr # global configurations +from indica.workflows.bayes_plots import plot_bayes_result + DEFAULT_PHANTOM_PARAMS = { "Ne_prof.y0": 5e19, "Ne_prof.wcenter": 0.4, @@ -73,9 +80,9 @@ OPTIMISED_PARAMS = [ # "Ne_prof.y1", "Ne_prof.y0", - "Ne_prof.peaking", - "Ne_prof.wcenter", - "Ne_prof.wped", + # "Ne_prof.peaking", + # "Ne_prof.wcenter", + # "Ne_prof.wped", # "ar_conc", # "Nimp_prof.y1", "Nimp_prof.y0", @@ -99,32 +106,40 @@ class TestWorkflow(BayesWorkflow): def __init__( self, pulse=None, + param_names=None, + opt_quantity=None, + nwalkers=50, tstart=0.02, tend=0.10, dt=0.01, tsample=0.06, iterations=100, - burn_in_fraction=0, + burn_frac=0, - phantom=False, + phantoms=False, diagnostics=None, ): self.pulse = pulse + self.param_names = param_names + self.opt_quantity = opt_quantity + self.tstart = tstart self.tend = tend self.dt = dt self.tsample = tsample self.nwalkers = nwalkers self.iterations = iterations - self.burn_in_fraction = burn_in_fraction + self.burn_frac = burn_frac - self.phantom = phantom + + self.phantoms = phantoms self.diagnostics = diagnostics self.setup_plasma() + self.save_phantom_profiles() self.read_data(diagnostics) - self.setup_opt_data(phantom=self.phantom) + self.setup_opt_data(phantoms=self.phantoms) self.setup_models(diagnostics) self.setup_optimiser() @@ -142,14 +157,13 @@ def setup_plasma(self): full_run=False, n_rad=20, ) - self.plasma.Ti_ref = True - self.plasma.time_to_calculate = self.plasma.t[ np.abs(self.tsample - self.plasma.t).argmin() ] self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) self.plasma.build_atomic_data(calc_power_loss=False) + def read_data(self, diagnostics: list): self.reader = ReadST40( self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt @@ -158,8 +172,8 @@ def read_data(self, diagnostics: list): self.plasma.set_equilibrium(self.reader.equilibrium) self.data = self.reader.binned_data - def setup_opt_data(self, phantom=False): - if phantom: + def setup_opt_data(self, phantoms=False): + if phantoms: self._phantom_data() else: self._exp_data() @@ -217,7 +231,6 @@ def setup_models(self, diagnostics: list): self.models[diag] = model - def setup_optimiser(self, ): self.bayesopt = BayesModels( @@ -243,7 +256,6 @@ def setup_optimiser(self, ): OPTIMISED_PARAMS, size=self.nwalkers ) - def _phantom_data(self, noise=False, noise_factor=0.1): self.opt_data = {} if "smmh1" in self.diagnostics: @@ -278,21 +290,28 @@ def _phantom_data(self, noise=False, noise_factor=0.1): "smmh1.ne" ].max().values * np.random.normal(0, noise_factor, None) self.opt_data["xrcs.spectra"] = self.opt_data[ - "xrcs.spectra" - ] + np.random.normal( + "xrcs.spectra" + ] + np.random.normal( 0, np.sqrt(self.opt_data["xrcs.spectra"].values[0,]), self.opt_data["xrcs.spectra"].shape[1], ) self.opt_data["cxff_pi.ti"] = self.opt_data[ - "cxff_pi.ti" - ] + self.opt_data["cxff_pi.ti"].max().values * np.random.normal( + "cxff_pi.ti" + ] + self.opt_data["cxff_pi.ti"].max().values * np.random.normal( 0, noise_factor, self.opt_data["cxff_pi.ti"].shape[1] ) self.opt_data["efit.wp"] = self.opt_data["efit.wp"] + self.opt_data[ "efit.wp" ].max().values * np.random.normal(0, noise_factor, None) + self.phantom_profiles = {} + for key in ["electron_density", "impurity_density", "electron_temperature", "ion_temperature", + "ion_density", "fast_density", "neutral_density"]: + self.phantom_profiles[key] = getattr(self.plasma, key).sel( + t=self.plasma.time_to_calculate + ).copy() + def _exp_data(self): self.opt_data = flatdict.FlatDict(self.data, ".") if "xrcs" in self.diagnostics: @@ -314,12 +333,12 @@ def _exp_data(self): self.opt_data["cxff_pi"]["ti"].channel == 2, drop=True ) - def __call__(self, *args, **kwargs): - self.sampler.run_mcmc(self.start_points, self.iterations, progress=True) - blobs = self.sampler.get_blobs( - discard=int(self.iterations * self.burn_in_fraction), flat=True - ) - return blobs + def __call__(self, filepath = "./results/test/", **kwargs): + self.run_sampler() + self.save_pickle(filepath=filepath) + plot_bayes_result(self.result, filepath) + return self.result + if __name__ == "__main__": @@ -329,8 +348,15 @@ def __call__(self, *args, **kwargs): tsample=0.060, diagnostics=["efit", "smmh1", "cxff_pi"], - iterations=10, - nwalkers=20, - burn_in_fraction=0.10, + iterations=20, + nwalkers=10, + burn_frac=0.10, + param_names = OPTIMISED_PARAMS, + opt_quantity=OPTIMISED_QUANTITY ) - run() + + test = run(filepath="./results/test/") + + flattest = flatdict.FlatDict(test, delimiter=".") + for key in flattest.keys(): + print(key) From dd3abdf8e2f82b6606cb716f2088497fafba44f0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 15:10:05 +0100 Subject: [PATCH 065/288] moved sampling to it's own function in abstract class --- indica/workflows/abstract_bayes_workflow.py | 42 ++++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index fdf2d5bc..6f6a3af4 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -1,5 +1,9 @@ from abc import ABC, abstractmethod import numpy as np +import xarray as xr +import pandas as pd +from pathlib import Path +import pickle class BayesWorkflow(ABC): def __init__(self, @@ -155,9 +159,43 @@ def _build_result_dict(self): self.result = result return self.result + def run_sampler(self): + + self.autocorr = sample_with_autocorr( + self.sampler, self.start_points, self.iterations, self.param_names.__len__(), auto_sample=10 + ) + blobs = self.sampler.get_blobs( + discard=int(self.iterations * self.burn_frac), flat=True + ) + blob_names = self.sampler.get_blobs().flatten()[0].keys() + self.samples = np.arange(0, blobs.__len__()) + + self.blobs = { + blob_name: xr.concat( + [data[blob_name] for data in blobs], + dim=pd.Index(self.samples, name="index"), + ) + for blob_name in blob_names + } + self.accept_frac = self.sampler.acceptance_fraction.sum() + self.prior_sample = self.bayesopt.sample_from_priors( + self.param_names, size=int(1e4) + ) + self.post_sample = self.sampler.get_chain(flat=True) + self.result = self._build_result_dict() + + + def save_pickle(self, filepath): + if filepath: + Path(filepath).mkdir(parents=True, exist_ok=True) + with open(filepath + "results.pkl", "wb") as handle: + pickle.dump(self.result, handle) + @abstractmethod - def __call__(self, *args, **kwargs): - return {} + def __call__(self, filepath="./results/test/", **kwargs): + self.run_sampler() + self.save_pickle(filepath=filepath) + return self.result def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sample=5): From 6f9d4a83cba071a719ac9108605c7209bba05383 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 15:18:53 +0100 Subject: [PATCH 066/288] renamed abstract class to AbstractBayesWorkflow --- indica/workflows/abstract_bayes_workflow.py | 2 +- indica/workflows/bayes_workflow_example.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 6f6a3af4..4f426d48 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -5,7 +5,7 @@ from pathlib import Path import pickle -class BayesWorkflow(ABC): +class AbstractBayesWorkflow(ABC): def __init__(self, phantom, diagnostics): diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 4bc2e39a..569d23f1 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -17,7 +17,7 @@ from indica.models.plasma import Plasma from indica.converters.line_of_sight import LineOfSightTransform -from abstract_bayes_workflow import BayesWorkflow, sample_with_autocorr +from abstract_bayes_workflow import AbstractBayesWorkflow # global configurations from indica.workflows.bayes_plots import plot_bayes_result @@ -102,7 +102,7 @@ OPTIMISED_QUANTITY = ["cxff_pi.ti", "efit.wp", "smmh1.ne"] -class TestWorkflow(BayesWorkflow): +class TestWorkflow(AbstractBayesWorkflow): def __init__( self, pulse=None, From 55e2f8fe3d53465a7bb7f05fbf6abea8e09afeba Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 15:40:46 +0100 Subject: [PATCH 067/288] comments --- indica/workflows/abstract_bayes_workflow.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 4f426d48..c0025b88 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -161,6 +161,16 @@ def _build_result_dict(self): def run_sampler(self): + """ + TODO: unsure if keeping in abstract class is best practice + + Runs the sampler and saves certain attributes from the sampler + + Returns + + result in MDSPlus node formatting + + """ self.autocorr = sample_with_autocorr( self.sampler, self.start_points, self.iterations, self.param_names.__len__(), auto_sample=10 ) From d3bf315e2a13d7124d7dad82b3af68dd443d99d6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 15:58:54 +0100 Subject: [PATCH 068/288] added high density sampling as method --- indica/bayesmodels.py | 32 ++++++++++++++++++++ indica/workflows/bayes_workflow_example.py | 35 ++++++++++++++-------- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index fc791837..d6092f09 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -165,6 +165,38 @@ def sample_from_priors(self, param_names, size=10): samples = samples[:, 0:size] return samples.transpose() + def sample_from_high_density_region(self, param_names: list, sampler: object, nwalkers: int, nsamples = 100): + start_points = self.sample_from_priors( + param_names, size=nsamples + ) + + ln_prob, _ = sampler.compute_log_prob(start_points) + num_best_points = int(nsamples * 0.05) + index_best_start = np.argsort(ln_prob)[-num_best_points:] + best_start_points = start_points[index_best_start, :] + best_points_std = np.std(best_start_points, axis=0) + + # Passing samples through ln_prior and redrawing if they fail + samples = np.empty((param_names.__len__(), 0)) + while samples.size < param_names.__len__() * nwalkers: + sample = np.random.normal( + np.mean(best_start_points, axis=0), + best_points_std * 2, + size=(nwalkers * 5, len(param_names)), + ) + start = { + name: sample[:, idx] for idx, name in enumerate(param_names) + } + ln_prior = self._ln_prior(start) + # Convert from dictionary of arrays -> array, + # then filtering out where ln_prior is -infinity + accepted_samples = np.array(list(start.values()))[ + :, ln_prior != -np.inf + ] + samples = np.append(samples, accepted_samples, axis=1) + start_points = samples[:, 0: nwalkers].transpose() + return start_points + def ln_posterior(self, parameters: dict, **kwargs): """ Posterior probability given to optimisers diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 569d23f1..4386642a 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -102,12 +102,13 @@ OPTIMISED_QUANTITY = ["cxff_pi.ti", "efit.wp", "smmh1.ne"] -class TestWorkflow(AbstractBayesWorkflow): +class ExampleWorkflow(AbstractBayesWorkflow): def __init__( self, pulse=None, param_names=None, opt_quantity=None, + priors = None, nwalkers=50, tstart=0.02, @@ -119,10 +120,12 @@ def __init__( phantoms=False, diagnostics=None, + sample_high_density = False, ): self.pulse = pulse self.param_names = param_names self.opt_quantity = opt_quantity + self.priors = priors self.tstart = tstart self.tend = tend @@ -135,6 +138,7 @@ def __init__( self.phantoms = phantoms self.diagnostics = diagnostics + self.sample_high_density = sample_high_density self.setup_plasma() self.save_phantom_profiles() @@ -237,24 +241,26 @@ def setup_optimiser(self, ): plasma=self.plasma, data=self.opt_data, diagnostic_models=[*self.models.values()], - quant_to_optimise=OPTIMISED_QUANTITY, - priors=DEFAULT_PRIORS, + quant_to_optimise=self.opt_quantity, + priors=self.priors, ) - ndim = len(OPTIMISED_PARAMS) - + ndim = len(self.param_names) self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] self.sampler = emcee.EnsembleSampler( self.nwalkers, ndim, log_prob_fn=self.bayesopt.ln_posterior, - parameter_names=OPTIMISED_PARAMS, + parameter_names=self.param_names, moves=self.move, ) - self.start_points = self.bayesopt.sample_from_priors( - OPTIMISED_PARAMS, size=self.nwalkers - ) + if self.sample_high_density: + self.start_points = self.bayesopt.sample_from_high_density_region(self.param_names, self.sampler, self.nwalkers) + else: + self.start_points = self.bayesopt.sample_from_priors( + self.param_names, size=self.nwalkers + ) def _phantom_data(self, noise=False, noise_factor=0.1): self.opt_data = {} @@ -342,17 +348,20 @@ def __call__(self, filepath = "./results/test/", **kwargs): if __name__ == "__main__": - run = TestWorkflow( + run = ExampleWorkflow( pulse=10009, dt=0.005, tsample=0.060, diagnostics=["efit", "smmh1", "cxff_pi"], + param_names=OPTIMISED_PARAMS, + opt_quantity=OPTIMISED_QUANTITY, + priors=DEFAULT_PRIORS, + iterations=20, - nwalkers=10, + nwalkers=50, burn_frac=0.10, - param_names = OPTIMISED_PARAMS, - opt_quantity=OPTIMISED_QUANTITY + sample_high_density=True ) test = run(filepath="./results/test/") From 430a44bee56c472bc6b0ec3c35c40c38c6c2474b Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 16:18:04 +0100 Subject: [PATCH 069/288] added attributes to __init__ --- indica/workflows/abstract_bayes_workflow.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index c0025b88..fae9a283 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -6,17 +6,25 @@ import pickle class AbstractBayesWorkflow(ABC): + @abstractmethod def __init__(self, - phantom, - diagnostics): - - self.phantom = phantom + phantoms = None, + diagnostics = None, + param_names=None, + opt_quantity=None, + priors=None, + ): + + self.phantoms = phantoms self.diagnostics = diagnostics + self.param_names = param_names + self.opt_quantity = opt_quantity + self.priors = priors self.setup_plasma() self.save_phantom_profiles() self.read_data(self.diagnostics) - self.setup_opt_data(self.phantom) + self.setup_opt_data(self.phantoms) self.setup_models(self.diagnostics) self.setup_optimiser() From d633ae5e3fec78f2e203068e42b8fc363ebe9d7d Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 16:30:24 +0100 Subject: [PATCH 070/288] error handling __init__ --- indica/workflows/bayes_workflow_example.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 4386642a..b7677e18 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -109,6 +109,7 @@ def __init__( param_names=None, opt_quantity=None, priors = None, + diagnostics=None, nwalkers=50, tstart=0.02, @@ -119,27 +120,30 @@ def __init__( burn_frac=0, phantoms=False, - diagnostics=None, sample_high_density = False, ): + self.pulse = pulse self.param_names = param_names self.opt_quantity = opt_quantity self.priors = priors + self.diagnostics = diagnostics self.tstart = tstart self.tend = tend self.dt = dt self.tsample = tsample + self.nwalkers = nwalkers self.iterations = iterations self.burn_frac = burn_frac - - self.phantoms = phantoms - self.diagnostics = diagnostics self.sample_high_density = sample_high_density + for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics"]: + if getattr(self, attribute) is None: + raise ValueError(f"{attribute} needs to be defined") + self.setup_plasma() self.save_phantom_profiles() self.read_data(diagnostics) @@ -352,8 +356,8 @@ def __call__(self, filepath = "./results/test/", **kwargs): pulse=10009, dt=0.005, tsample=0.060, - diagnostics=["efit", "smmh1", "cxff_pi"], - + # diagnostics=["efit", "smmh1", "cxff_pi"], + diagnostics=None, param_names=OPTIMISED_PARAMS, opt_quantity=OPTIMISED_QUANTITY, priors=DEFAULT_PRIORS, From 35b64cf36f0d6918117cb239c5a0eac2c97db2b6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 16:30:37 +0100 Subject: [PATCH 071/288] error handling __init__ --- indica/workflows/bayes_workflow_dev.py | 469 +++++++++---------------- 1 file changed, 170 insertions(+), 299 deletions(-) diff --git a/indica/workflows/bayes_workflow_dev.py b/indica/workflows/bayes_workflow_dev.py index 5961bb5f..ac6af031 100644 --- a/indica/workflows/bayes_workflow_dev.py +++ b/indica/workflows/bayes_workflow_dev.py @@ -1,11 +1,7 @@ import emcee import numpy as np -import xarray as xr -import pandas as pd import flatdict from scipy.stats import loguniform -from pathlib import Path -import pickle from indica.readers.read_st40 import ReadST40 from indica.bayesmodels import BayesModels, get_uniform @@ -17,6 +13,7 @@ from indica.models.plasma import Plasma from indica.converters.line_of_sight import LineOfSightTransform +from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow from indica.writers.bda_tree import create_nodes, write_nodes # global configurations @@ -78,9 +75,9 @@ OPTIMISED_PARAMS = [ # "Ne_prof.y1", "Ne_prof.y0", - "Ne_prof.peaking", - "Ne_prof.wcenter", - "Ne_prof.wped", + # "Ne_prof.peaking", + # "Ne_prof.wcenter", + # "Ne_prof.wped", # "ar_conc", # "Nimp_prof.y1", "Nimp_prof.y0", @@ -100,140 +97,83 @@ OPTIMISED_QUANTITY = ["cxff_pi.ti", "efit.wp", "smmh1.ne"] -class BayesWorkflow: +class DevBayesWorkflow(AbstractBayesWorkflow): def __init__( - self, - pulse=None, - pulse_to_write = 23000101, - result_path="./results/example/", - nwalkers=50, - tstart=0.02, - tend=0.10, - dt=0.01, - tsample=0.06, - iterations=100, - burn_in_fraction=0, - - diagnostics=None, - center_mass_sampling=True, - Ti_ref=True, - profiles=None, - phantom=True, - mds_write=False, - save_plots=True, + self, + pulse=None, + pulse_to_write=None, + param_names=None, + opt_quantity=None, + priors = None, + + nwalkers=50, + tstart=0.02, + tend=0.10, + dt=0.01, + tsample=0.06, + iterations=100, + burn_frac=0, + + phantoms=False, + diagnostics=None, + sample_high_density = False, + mds_write=False, + fast_particles = False, ): self.pulse = pulse self.pulse_to_write = pulse_to_write + self.param_names = param_names + self.opt_quantity = opt_quantity + self.priors = priors + self.tstart = tstart self.tend = tend self.dt = dt self.tsample = tsample - self.result_path = result_path self.nwalkers = nwalkers self.iterations = iterations - self.burn_in_fraction = burn_in_fraction - self.center_mass_sampling = center_mass_sampling - self.Ti_ref = Ti_ref + self.burn_frac = burn_frac + self.phantoms = phantoms self.diagnostics = diagnostics - self.phantom = phantom - self.profiles = profiles + self.sample_high_density = sample_high_density self.mds_write = mds_write - self.save_plots = save_plots + self.fast_particles = fast_particles + + for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics"]: + if getattr(self, attribute) is None: + raise ValueError(f"{attribute} needs to be defined") + + self.setup_plasma() + self.save_phantom_profiles() + self.read_data(diagnostics) + self.setup_opt_data(phantoms=self.phantoms) + self.setup_models(diagnostics) + self.setup_optimiser() + def setup_plasma(self): self.plasma = Plasma( - tstart=tstart, - tend=tend, - dt=dt, + tstart=self.tstart, + tend=self.tend, + dt=self.dt, main_ion="h", impurities=("ar", "c"), impurity_concentration=( 0.001, - 0.04, + 0.02, ), full_run=False, n_rad=20, ) - self.plasma.Ti_ref = self.Ti_ref - self.tsample = tsample self.plasma.time_to_calculate = self.plasma.t[ - np.abs(tsample - self.plasma.t).argmin() + np.abs(self.tsample - self.plasma.t).argmin() ] self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) self.plasma.build_atomic_data(calc_power_loss=False) - self.init_fast_particles() - - self.read_st40(diagnostics) - - self.init_models() - - if self.phantom: - self.phantom_data() - self.save_profiles() - else: - self.exp_data() - - self.bayes_run = BayesModels( - plasma=self.plasma, - data=self.flat_data, - diagnostic_models=[*self.models.values()], - quant_to_optimise=OPTIMISED_QUANTITY, - priors=DEFAULT_PRIORS, - ) - - ndim = len(OPTIMISED_PARAMS) - self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] - self.sampler = emcee.EnsembleSampler( - nwalkers, - ndim, - log_prob_fn=self.bayes_run.ln_posterior, - parameter_names=OPTIMISED_PARAMS, - moves=self.move, - # kwargs={ - # "moment_analysis": False, - # "calc_spectra": True, - # "minimum_lines": False, - # "background": self.flat_data["xrcs.background"], - # }, - ) + if self.fast_particles: + self._init_fast_particles() - if not self.center_mass_sampling: - self.start_points = self.bayes_run.sample_from_priors( - OPTIMISED_PARAMS, size=self.nwalkers - ) - else: - # gaussian mass around most probable starting points TODO: move this function to it's own method - nsamples = 100 - start_points = self.bayes_run.sample_from_priors( - OPTIMISED_PARAMS, size=nsamples - ) - ln_prob, _ = self.sampler.compute_log_prob(start_points) - num_best_points = int(nsamples * 0.05) - index_best_start = np.argsort(ln_prob)[-num_best_points:] - best_start_points = start_points[index_best_start, :] - best_points_std = np.std(best_start_points, axis=0) - - # Passing samples through ln_prior and redrawing if they fail - samples = np.empty((OPTIMISED_PARAMS.__len__(), 0)) - while samples.size < OPTIMISED_PARAMS.__len__() * self.nwalkers: - sample = np.random.normal( - np.mean(best_start_points, axis=0), - best_points_std * 2, - size=(self.nwalkers * 5, len(OPTIMISED_PARAMS)), - ) - start = { - name: sample[:, idx] for idx, name in enumerate(OPTIMISED_PARAMS) - } - ln_prior = self.bayes_run._ln_prior(start) - # Convert from dictionary of arrays -> array, - # then filtering out where ln_prior is -infinity - accepted_samples = np.array(list(start.values()))[ - :, ln_prior != -np.inf - ] - samples = np.append(samples, accepted_samples, axis=1) - self.start_points = samples[:, 0 : self.nwalkers].transpose() - - def init_fast_particles(self): + def _init_fast_particles(self): st40_code = ReadST40(13110009, self.tstart, self.tend, dt=self.dt, tree="astra") st40_code.get_raw_data("", "astra", "RUN573") st40_code.bin_data_in_time(["astra"], self.tstart, self.tend, self.dt) @@ -253,41 +193,25 @@ def init_fast_particles(self): Pbper = code_data["pbper"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) self.plasma.pressure_fast_perpendicular.values = Pbper.values - def read_st40(self, diagnostics=None): - self.ST40_data = ReadST40( + def read_data(self, diagnostics: list): + self.reader = ReadST40( self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt ) - self.ST40_data(diagnostics) - self.plasma.set_equilibrium(self.ST40_data.equilibrium) + self.reader(diagnostics) + self.plasma.set_equilibrium(self.reader.equilibrium) + self.data = self.reader.binned_data - def save_profiles(self): - phantom_profiles = { - "electron_density": self.plasma.electron_density.sel( - t=self.plasma.time_to_calculate - ).copy(), - "impurity_density": self.plasma.impurity_density.sel( - t=self.plasma.time_to_calculate - ).copy(), - "electron_temperature": self.plasma.electron_temperature.sel( - t=self.plasma.time_to_calculate - ).copy(), - "ion_temperature": self.plasma.ion_temperature.sel( - t=self.plasma.time_to_calculate - ).copy(), - "fast_density": self.plasma.fast_density.sel( - t=self.plasma.time_to_calculate - ).copy(), - "neutral_density": self.plasma.neutral_density.sel( - t=self.plasma.time_to_calculate - ).copy(), - } - self.profiles = phantom_profiles + def setup_opt_data(self, phantoms=False): + if phantoms: + self._phantom_data() + else: + self._exp_data() - def init_models(self): + def setup_models(self, diagnostics: list): self.models = {} for diag in self.diagnostics: if diag == "smmh1": - # los_transform = self.ST40_data.binned_data["smmh1"]["ne"].transform + # los_transform = self.data["smmh1"]["ne"].transform machine_dims = self.plasma.machine_dimensions origin = np.array([[-0.38063365, 0.91893092, 0.01]]) # end = np.array([[0, 0, 0.01]]) @@ -304,222 +228,169 @@ def init_models(self): passes=2, ) los_transform.set_equilibrium(self.plasma.equilibrium) - model = Interferometry(name="smmh1") + model = Interferometry(name=diag) model.set_los_transform(los_transform) model.plasma = self.plasma - self.models["smmh1"] = model - if "xrcs" in self.diagnostics: - los_transform = self.ST40_data.binned_data["xrcs"]["te_kw"].transform + elif diag == "xrcs": + los_transform = self.data["xrcs"]["te_kw"].transform model = Helike_spectroscopy( name="xrcs", window_masks=[slice(0.394, 0.396)], - window_vector=self.ST40_data.binned_data["xrcs"][ - "spectra" - ].wavelength.values - * 0.1, + window_vector=self.data[diag][ + "spectra" + ].wavelength.values + * 0.1, ) model.set_los_transform(los_transform) model.plasma = self.plasma - self.models["xrcs"] = model - if "efit" in self.diagnostics: + elif diag == "efit": model = EquilibriumReconstruction(name="efit") model.plasma = self.plasma - self.models["efit"] = model - if "cxff_pi" in self.diagnostics: - transform = self.ST40_data.binned_data["cxff_pi"]["ti"].transform - transform.set_equilibrium(self.ST40_data.equilibrium) - model = ChargeExchange(name="cxff_pi", element="ar") + elif diag == "cxff_pi": + transform = self.data[diag]["ti"].transform + transform.set_equilibrium(self.plasma.equilibrium) + model = ChargeExchange(name=diag, element="ar") model.set_transect_transform(transform) model.plasma = self.plasma - self.models["cxff_pi"] = model + else: + raise ValueError(f"{diag} not found in setup_models") + + self.models[diag] = model + + def setup_optimiser(self, ): + + self.bayesopt = BayesModels( + plasma=self.plasma, + data=self.opt_data, + diagnostic_models=[*self.models.values()], + quant_to_optimise=self.opt_quantity, + priors=self.priors, + ) + + ndim = len(self.param_names) + self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] + self.sampler = emcee.EnsembleSampler( + self.nwalkers, + ndim, + log_prob_fn=self.bayesopt.ln_posterior, + parameter_names=self.param_names, + moves=self.move, + ) - def phantom_data(self, noise=False, noise_factor=0.1): - self.flat_data = {} + if self.sample_high_density: + self.start_points = self.bayesopt.sample_from_high_density_region(self.param_names, self.sampler, self.nwalkers) + else: + self.start_points = self.bayesopt.sample_from_priors( + self.param_names, size=self.nwalkers + ) + + def _phantom_data(self, noise=False, noise_factor=0.1): + self.opt_data = {} if "smmh1" in self.diagnostics: - self.flat_data["smmh1.ne"] = ( + self.opt_data["smmh1.ne"] = ( self.models["smmh1"]() - .pop("ne") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("ne") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if "xrcs" in self.diagnostics: - self.flat_data["xrcs.spectra"] = ( + self.opt_data["xrcs.spectra"] = ( self.models["xrcs"]() - .pop("spectra") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("spectra") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) - self.flat_data["xrcs.background"] = None + self.opt_data["xrcs.background"] = None if "cxff_pi" in self.diagnostics: cxrs_data = ( self.models["cxff_pi"]() - .pop("ti") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("ti") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) - self.flat_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) + self.opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) if "efit" in self.diagnostics: - self.flat_data["efit.wp"] = ( + self.opt_data["efit.wp"] = ( self.models["efit"]() - .pop("wp") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("wp") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if noise: - self.flat_data["smmh1.ne"] = self.flat_data["smmh1.ne"] + self.flat_data[ + self.opt_data["smmh1.ne"] = self.opt_data["smmh1.ne"] + self.opt_data[ "smmh1.ne" ].max().values * np.random.normal(0, noise_factor, None) - self.flat_data["xrcs.spectra"] = self.flat_data[ - "xrcs.spectra" - ] + np.random.normal( + self.opt_data["xrcs.spectra"] = self.opt_data[ + "xrcs.spectra" + ] + np.random.normal( 0, - np.sqrt(self.flat_data["xrcs.spectra"].values[0,]), - self.flat_data["xrcs.spectra"].shape[1], + np.sqrt(self.opt_data["xrcs.spectra"].values[0,]), + self.opt_data["xrcs.spectra"].shape[1], ) - self.flat_data["cxff_pi.ti"] = self.flat_data[ - "cxff_pi.ti" - ] + self.flat_data["cxff_pi.ti"].max().values * np.random.normal( - 0, noise_factor, self.flat_data["cxff_pi.ti"].shape[1] + self.opt_data["cxff_pi.ti"] = self.opt_data[ + "cxff_pi.ti" + ] + self.opt_data["cxff_pi.ti"].max().values * np.random.normal( + 0, noise_factor, self.opt_data["cxff_pi.ti"].shape[1] ) - self.flat_data["efit.wp"] = self.flat_data["efit.wp"] + self.flat_data[ + self.opt_data["efit.wp"] = self.opt_data["efit.wp"] + self.opt_data[ "efit.wp" ].max().values * np.random.normal(0, noise_factor, None) - def exp_data(self): - self.flat_data = flatdict.FlatDict(self.ST40_data.binned_data, ".") + self.phantom_profiles = {} + for key in ["electron_density", "impurity_density", "electron_temperature", "ion_temperature", + "ion_density", "fast_density", "neutral_density"]: + self.phantom_profiles[key] = getattr(self.plasma, key).sel( + t=self.plasma.time_to_calculate + ).copy() + + def _exp_data(self): + self.opt_data = flatdict.FlatDict(self.data, ".") if "xrcs" in self.diagnostics: - self.flat_data["xrcs.spectra"]["wavelength"] = ( - self.flat_data["xrcs.spectra"].wavelength * 0.1 + self.opt_data["xrcs.spectra"]["wavelength"] = ( + self.opt_data["xrcs.spectra"].wavelength * 0.1 ) - background = self.flat_data["xrcs.spectra"].where( - (self.flat_data["xrcs.spectra"].wavelength < 0.392) - & (self.flat_data["xrcs.spectra"].wavelength > 0.388), + background = self.opt_data["xrcs.spectra"].where( + (self.opt_data["xrcs.spectra"].wavelength < 0.392) + & (self.opt_data["xrcs.spectra"].wavelength > 0.388), drop=True, ) - self.flat_data["xrcs.background"] = background.mean(dim="wavelength") - self.flat_data["xrcs.spectra"]["error"] = np.sqrt( - self.flat_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 + self.opt_data["xrcs.background"] = background.mean(dim="wavelength") + self.opt_data["xrcs.spectra"]["error"] = np.sqrt( + self.opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 ) if "cxff_pi" in self.diagnostics: - self.flat_data["cxff_pi"]["ti"] = self.flat_data["cxff_pi"]["ti"].where( - self.flat_data["cxff_pi"]["ti"].channel == 2, drop=True - ) - - def __call__(self, *args, **kwargs): - autocorr = sample_with_autocorr( - self.sampler, self.start_points, self.iterations, auto_sample=10 - ) - blobs = self.sampler.get_blobs( - discard=int(self.iterations * self.burn_in_fraction), flat=True - ) - blob_names = self.sampler.get_blobs().flatten()[0].keys() - blob_dict = { - blob_name: xr.concat( - [data[blob_name] for data in blobs], - dim=pd.Index(np.arange(0, blobs.__len__()), name="index"), + self.opt_data["cxff_pi"]["ti"] = self.opt_data["cxff_pi"]["ti"].where( + self.opt_data["cxff_pi"]["ti"].channel == 2, drop=True ) - for blob_name in blob_names - } - samples = self.sampler.get_chain(flat=True) - - prior_samples = self.bayes_run.sample_from_priors( - OPTIMISED_PARAMS, size=int(1e4) - ) - self.results = { - "blobs": blob_dict, - "diag_data": self.flat_data, - "samples": samples, - "prior_samples": prior_samples, - "param_names": OPTIMISED_PARAMS, - "phantom_profiles": self.profiles, - "autocorr": autocorr, - "acceptance_fraction": self.sampler.acceptance_fraction.sum(), - } - - self.nested_results = { - "DIAG_DATA":{"efit":{"wp":1}}, - # self.ST40_data.binned_data["efit"]["wp"].values[4,] - # "MODEL_DATA": {}, - "METADATA": { - "DIAG_RUNS": "EMPTY", - "EQUIL": self.ST40_data.equilibrium.__str__(), - "IMP1": self.plasma.impurities[0], - "MAIN_ION": self.plasma.main_ion, - }, - "OPTIMISATION": { - "AUTO_CORR":autocorr, - "BURN_FRAC":self.burn_in_fraction, - "ITER": self.iterations, - "NWALKERS": self.nwalkers, - "PARAM_NAMES": OPTIMISED_PARAMS, - "POST_SAMPLE": samples, - "PRIOR_SAMPLE": prior_samples - }, - # "PHANTOMS": self.profiles, - # "PROFILES": {}, - # "TIME": self.plasma.t, - # "TIME_OPT": self.plasma.time_to_calculate - - } - - # self.acceptance_fraction = self.sampler.acceptance_fraction.sum() - print(self.results["acceptance_fraction"]) - - Path(self.result_path).mkdir(parents=True, exist_ok=True) - with open(self.result_path + "results.pkl", "wb") as handle: - pickle.dump(self.results, handle) - - if self.save_plots: - plot_bayes_result(self.results, figheader=self.result_path, filetype=".png") - - -def sample_with_autocorr(sampler, start_points, iterations, auto_sample=5): - autocorr = np.ones((iterations,)) * np.nan - old_tau = np.inf - for sample in sampler.sample( - start_points, - iterations=iterations, - progress=True, - ): - if sampler.iteration % auto_sample: - continue - new_tau = np.mean(sampler.get_autocorr_time(tol=0)) - autocorr[sampler.iteration - 1] = new_tau - converged = np.all(new_tau * 50 < sampler.iteration) - converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) - if converged: - break - old_tau = new_tau - autocorr = autocorr[: sampler.iteration, ] - return autocorr - + def __call__(self, filepath = "./results/test/", **kwargs): + self.run_sampler() + self.save_pickle(filepath=filepath) + plot_bayes_result(self.result, filepath) + return self.result if __name__ == "__main__": - mds_write=False - pulse_to_write = 23000101 - if mds_write: - node_structure = create_nodes(pulse_to_write=pulse_to_write, diagnostic_quantities=OPTIMISED_QUANTITY) + # mds_write=False + # pulse_to_write = 23000101 + # if mds_write: + # node_structure = create_nodes(pulse_to_write=pulse_to_write, diagnostic_quantities=OPTIMISED_QUANTITY) - run = BayesWorkflow( + run = DevBayesWorkflow( pulse=10009, - result_path="./results/test/", iterations=20, nwalkers=20, - burn_in_fraction=0.10, + burn_frac=0.10, dt=0.005, tsample=0.060, diagnostics=["efit", "smmh1", "cxff_pi"], - phantom=False, - center_mass_sampling=False, - Ti_ref=True, - mds_write=mds_write, - save_plots=True + phantoms=False, + sample_high_density=True, + mds_write=False, ) - run() + results = run(filepath="./results/test/",) - if mds_write: - write_nodes(pulse_to_write, node_structure, run.nested_results) + # if mds_write: + # write_nodes(pulse_to_write, node_structure, run.nested_results) From 633b36c7d13dc6c61a7b75c733784f92ee1837ff Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 4 Jul 2023 16:40:25 +0100 Subject: [PATCH 072/288] moved read data to abstract class --- indica/workflows/abstract_bayes_workflow.py | 19 ++++++++++--------- indica/workflows/bayes_workflow_dev.py | 8 -------- indica/workflows/bayes_workflow_example.py | 12 +----------- 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index fae9a283..1faae4e6 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -5,6 +5,9 @@ from pathlib import Path import pickle +from indica.readers.read_st40 import ReadST40 + + class AbstractBayesWorkflow(ABC): @abstractmethod def __init__(self, @@ -35,16 +38,14 @@ def setup_plasma(self): """ self.plasma = None - @abstractmethod - def read_data(self, diagnostics: list): - """ - Reads data from server - - Returns - nested dictionary of data - """ - self.data = {} + def read_data(self, diagnostics: list): + self.reader = ReadST40( + self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt + ) + self.reader(diagnostics) + self.plasma.set_equilibrium(self.reader.equilibrium) + self.data = self.reader.binned_data @abstractmethod diff --git a/indica/workflows/bayes_workflow_dev.py b/indica/workflows/bayes_workflow_dev.py index ac6af031..28c228ed 100644 --- a/indica/workflows/bayes_workflow_dev.py +++ b/indica/workflows/bayes_workflow_dev.py @@ -193,14 +193,6 @@ def _init_fast_particles(self): Pbper = code_data["pbper"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) self.plasma.pressure_fast_perpendicular.values = Pbper.values - def read_data(self, diagnostics: list): - self.reader = ReadST40( - self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt - ) - self.reader(diagnostics) - self.plasma.set_equilibrium(self.reader.equilibrium) - self.data = self.reader.binned_data - def setup_opt_data(self, phantoms=False): if phantoms: self._phantom_data() diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index b7677e18..1fd9d31e 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -171,15 +171,6 @@ def setup_plasma(self): self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) self.plasma.build_atomic_data(calc_power_loss=False) - - def read_data(self, diagnostics: list): - self.reader = ReadST40( - self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt - ) - self.reader(diagnostics) - self.plasma.set_equilibrium(self.reader.equilibrium) - self.data = self.reader.binned_data - def setup_opt_data(self, phantoms=False): if phantoms: self._phantom_data() @@ -356,8 +347,7 @@ def __call__(self, filepath = "./results/test/", **kwargs): pulse=10009, dt=0.005, tsample=0.060, - # diagnostics=["efit", "smmh1", "cxff_pi"], - diagnostics=None, + diagnostics=["efit", "smmh1", "cxff_pi"], param_names=OPTIMISED_PARAMS, opt_quantity=OPTIMISED_QUANTITY, priors=DEFAULT_PRIORS, From 2ce1c5893724d5c4894f3900a046024da1b7e613 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 7 Jul 2023 10:07:46 +0100 Subject: [PATCH 073/288] initial commit --- indica/writers/bda_tree.py | 225 +++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 indica/writers/bda_tree.py diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py new file mode 100644 index 00000000..66126ee2 --- /dev/null +++ b/indica/writers/bda_tree.py @@ -0,0 +1,225 @@ +# Create trees from ppac standard utility tools +import standard_utility as util +from MDSplus import Connection +import sys +import numpy as np + + +def bda(): + nodes = { + "TIME": ("NUMERIC", "time vector, s"), + "TIME_OPT": ("NUMERIC", "time of optimisation, s"), + "METADATA": { + "GITCOMMIT": ("TEXT", "Commit ID used for run"), + "USER": ("TEXT", "Username of owner"), + "PULSE": ("NUMERIC", "Pulse number analysed"), + "EQUIL": ("TEXT", "Equilibrium used"), + "MAIN_ION": ("TEXT", "Main ion element"), + "IMP1": ("TEXT", "Impurity element chosen for Z1"), + "IMP2": ("TEXT", "Impurity element chosen for Z2"), + "DIAG_RUNS": ("TEXT", "RUNS from which diagnostic data originated") + }, + "PROFILES": { + "RHO_POLOIDAL": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "NE": ("SIGNAL", "Electron density, m^-3"), + "NI": ("SIGNAL", "Ion density, m^-3"), + "TE": ("SIGNAL", "Electron temperature, eV"), + "TI": ("SIGNAL", "Ion temperature of main ion, eV"), + "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), + "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), + "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), + "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), + + + "TIZ1": ("SIGNAL", "Ion temperature of impurity IMP1, eV"), + "TIZ2": ("SIGNAL", "Ion temperature of impurity IMP2, eV"), + "TIZ3": ("SIGNAL", "Ion temperature of impurity IMP3, eV"), + "NIZ1": ("SIGNAL", "Density of impurity IMP1, m^-3"), + "NIZ2": ("SIGNAL", "Density of impurity IMP2, m^-3"), + "NIZ3": ("SIGNAL", "Density of impurity IMP3, m^-3"), + "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), + "ZI": ("SIGNAL", "Average charge of main ion, "), + "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), + "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), + "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), + "ZEFF": ("SIGNAL", "Effective charge, "), + "P": ("SIGNAL", "Pressure,Pa"), + "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), + }, + + "PROFILE_STAT": { + "SAMPLES": ("NUMERIC", "Numerical index of the optimisation samples"), + "RHO_POLOIDAL": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "NE": ("SIGNAL", "Electron density, m^-3"), + "NI": ("SIGNAL", "Ion density, m^-3"), + "TE": ("SIGNAL", "Electron temperature, eV"), + "TI": ("SIGNAL", "Ion temperature of main ion, eV"), + + "TIZ1": ("SIGNAL", "Ion temperature of impurity IMP1, eV"), + "TIZ2": ("SIGNAL", "Ion temperature of impurity IMP2, eV"), + "TIZ3": ("SIGNAL", "Ion temperature of impurity IMP3, eV"), + "NIZ1": ("SIGNAL", "Density of impurity IMP1, m^-3"), + "NIZ2": ("SIGNAL", "Density of impurity IMP2, m^-3"), + "NIZ3": ("SIGNAL", "Density of impurity IMP3, m^-3"), + "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), + "ZI": ("SIGNAL", "Average charge of main ion, "), + "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), + "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), + "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), + "ZEFF": ("SIGNAL", "Effective charge, "), + "P": ("SIGNAL", "Pressure,Pa"), + "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), + }, + + "GLOBAL": { + "NE0": ("SIGNAL", "Central electron density, m^-3"), + "NI0": ("SIGNAL", "Central ion density, m^-3"), + "TE0": ("SIGNAL", "Central electron temperature, eV"), + "TI0": ("SIGNAL", "Central ion temperature of main ion, eV"), + + "NE0_ERR": ("SIGNAL", "Central electron density error, m^-3"), + "NI0_ERR": ("SIGNAL", "Central ion density error, m^-3"), + "TE0_ERR": ("SIGNAL", "Central electron temperature error, eV"), + "TI0_ERR": ("SIGNAL", "Central ion temperature of main ion error, eV"), + + }, + "PHANTOMS": { + "FLAG": ("TEXT", "True if phantoms used"), + "RHO_POLOIDAL": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "NE": ("SIGNAL", "Electron density, m^-3"), + "NI": ("SIGNAL", "Ion density, m^-3"), + "TE": ("SIGNAL", "Electron temperature, eV"), + "TI": ("SIGNAL", "Ion temperature of main ion, eV"), + # "NIMP1": ("SIGNAL", "Impurity density, m^-3"), + + }, + + "INPUT": { + "PARAM_NAMES": ("TEXT", "Names of params optimised"), + "MODEL_KWARGS": ("TEXT", "Model key word arguments"), + "OPT_KWARGS": ("TEXT", "Optimiser key word arguments"), + "ITER": ("NUMERIC", "Iterations of optimiser"), + "NWALKERS": ("NUMERIC", "Number of walkers used"), + "BURN_FRAC": ("NUMERIC", "Burn in fraction for chains"), + }, + + "OPTIMISATION": { + "AUTO_CORR": ("NUMERIC", "Auto-correlation (iteration nwalker)"), + "POST_SAMPLE": ("NUMERIC", "Posterior probability samples (sample)"), + "PRIOR_SAMPLE": ("NUMERIC", "Prior samples"), + }, + } + return nodes + + +DIAGNOSTIC_QUANTITY = ["DIAGNOSTIC1.QUANTITY1", "DIAGNOSTIC1.QUANTITY2", "DIAGNOSTIC2.QUANTITY1"] + + +def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_quantities=DIAGNOSTIC_QUANTITY): + bda_nodes = bda() + quant_list = [item.split(".") for item in diagnostic_quantities] # replace OPTIMISED_QUANTITY + diag_names = list(set([item[0] for item in quant_list])) + + + diag_nodes = {diag_name: + {quantity[1]: ("SIGNAL", f"measured {quantity[1]} from {quantity[0]}") + for quantity in quant_list if quantity[0] == diag_name} + for diag_name in diag_names + } + for diag_name in diag_names: + diag_nodes[diag_name]["RUN"] = ("TEXT", f"RUN from which {diag_name} data was taken") + + + nodes = { + "RUN": ("TEXT", "RUN used for diagnostic"), + "USAGE": ("TEXT", "Diagnostic used in analysis"), + "PULSE": ("NUMERIC", "Pulse used for diagnostic") + } + + workflow_nodes = {diag_name: nodes + for diag_name in diag_names + } + + + model_nodes = {diag_name: + {quantity[1]: ("SIGNAL", f"modelled {quantity[1]} from {quantity[0]}") + for quantity in quant_list if quantity[0] == diag_name} + for diag_name in diag_names + } + model_nodes["SAMPLES"] = ("NUMERIC", "index of samples taken from optimisation") + bda_nodes["MODEL_DATA"] = model_nodes + bda_nodes["DIAG_DATA"] = diag_nodes + bda_nodes["INPUT"]["WORKFLOW"] = workflow_nodes + + tree = "ST40" + script_name = "BDA" + script_info = "Bayesian Diagnostic Analysis" + + node_info = util.GetNodeInformation( + script=None, + node_information_type="json", + run_name=run, + run_info=run, + script_name=script_name, + script_info=script_info, + root_node=None, + tree=tree, + pulse_number=pulse_to_write, + base_node_to_read=None, + node_information_file=bda_nodes + ).get() + + util.StandardNodeCreation( + pulse_number=pulse_to_write, + dict_node_info=node_info, + mode="EDIT", + name_of_BEST="BEST", # name of the structure linked to BEST + link_BEST_to_run=best, + ) + return node_info + + +def write_nodes(pulse, node_info, data): + util.StandardNodeWriting( + pulse_number=pulse, # pulse number for which data should be written to the structure + dict_node_info=node_info, # node information file + nodes_to_write=[], # selective nodes to be written + data_to_write=data, + debug=False, + ) + + +def check_analysis_run(pulseNo, which_run, ): + # Checker function to see if data already exists in a run + IP_address_smaug = '192.168.1.7:8000' + conn = Connection(IP_address_smaug) + conn.openTree('BDA', pulseNo) + + temp = conn.get('BDA.' + which_run + '.TIME').data() + conn.closeAllTrees() + + overwrite_flag = True + if isinstance(temp, np.ndarray): + print(f'Data already Exists in pulseNo = {pulseNo}, which_run = {which_run}') + print('User prompt...') + question = f' Scheduled to overwrite pulseNo {pulseNo}, {which_run}' \ + f'\n Do you want to overwrite {which_run}? (y/n)' + overwrite_flag = query_yes_no(question) + return overwrite_flag + + +def query_yes_no(question, ): + valid = {"yes": True, "y": True, "no": False, "n": False} + while True: + sys.stdout.write(question) + choice = input().lower() + if choice == "": + return False + elif choice in valid.keys(): + return valid[choice] + else: + sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") + + +if __name__ == "__main__": + create_nodes(pulse_to_write=23000101) From 08cf081a7fd05975de0c2e88f7ce3d0de4aac36d Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 7 Jul 2023 11:36:36 +0100 Subject: [PATCH 074/288] kwargs added --- indica/workflows/bayes_workflow_dev.py | 27 +++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/indica/workflows/bayes_workflow_dev.py b/indica/workflows/bayes_workflow_dev.py index 28c228ed..d82eb87d 100644 --- a/indica/workflows/bayes_workflow_dev.py +++ b/indica/workflows/bayes_workflow_dev.py @@ -94,7 +94,11 @@ # "Ti_prof.peaking", ] -OPTIMISED_QUANTITY = ["cxff_pi.ti", "efit.wp", "smmh1.ne"] +OPTIMISED_QUANTITY = [ + # "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne"] class DevBayesWorkflow(AbstractBayesWorkflow): @@ -105,6 +109,7 @@ def __init__( param_names=None, opt_quantity=None, priors = None, + phantom_params=None, nwalkers=50, tstart=0.02, @@ -125,6 +130,7 @@ def __init__( self.param_names = param_names self.opt_quantity = opt_quantity self.priors = priors + self.phantom_params = phantom_params self.tstart = tstart self.tend = tend @@ -140,7 +146,7 @@ def __init__( self.mds_write = mds_write self.fast_particles = fast_particles - for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics"]: + for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics", "phantom_params"]: if getattr(self, attribute) is None: raise ValueError(f"{attribute} needs to be defined") @@ -168,7 +174,7 @@ def setup_plasma(self): self.plasma.time_to_calculate = self.plasma.t[ np.abs(self.tsample - self.plasma.t).argmin() ] - self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) + self.plasma.update_profiles(self.phantom_params) self.plasma.build_atomic_data(calc_power_loss=False) if self.fast_particles: self._init_fast_particles() @@ -222,7 +228,6 @@ def setup_models(self, diagnostics: list): los_transform.set_equilibrium(self.plasma.equilibrium) model = Interferometry(name=diag) model.set_los_transform(los_transform) - model.plasma = self.plasma elif diag == "xrcs": los_transform = self.data["xrcs"]["te_kw"].transform @@ -235,20 +240,19 @@ def setup_models(self, diagnostics: list): * 0.1, ) model.set_los_transform(los_transform) - model.plasma = self.plasma elif diag == "efit": model = EquilibriumReconstruction(name="efit") - model.plasma = self.plasma elif diag == "cxff_pi": transform = self.data[diag]["ti"].transform transform.set_equilibrium(self.plasma.equilibrium) model = ChargeExchange(name=diag, element="ar") model.set_transect_transform(transform) - model.plasma = self.plasma else: raise ValueError(f"{diag} not found in setup_models") + model.plasma = self.plasma + self.models[diag] = model @@ -372,15 +376,20 @@ def __call__(self, filepath = "./results/test/", **kwargs): run = DevBayesWorkflow( pulse=10009, - iterations=20, + iterations=10, nwalkers=20, burn_frac=0.10, dt=0.005, tsample=0.060, - diagnostics=["efit", "smmh1", "cxff_pi"], + diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], phantoms=False, sample_high_density=True, mds_write=False, + + opt_quantity=OPTIMISED_QUANTITY, + param_names=OPTIMISED_PARAMS, + phantom_params=DEFAULT_PHANTOM_PARAMS, + priors=DEFAULT_PRIORS, ) results = run(filepath="./results/test/",) From 82eb00b40c27101f64a3d0597c0282b0bb8417c6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 7 Jul 2023 14:24:25 +0100 Subject: [PATCH 075/288] xrcs wavelength units corrected --- indica/workflows/bayes_workflow_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 1fd9d31e..ea954787 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -210,7 +210,7 @@ def setup_models(self, diagnostics: list): window_vector=self.data[diag][ "spectra" ].wavelength.values - * 0.1, + , ) model.set_los_transform(los_transform) model.plasma = self.plasma From 666387bb7631fcccc014b1a5b680e84921a50c10 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 7 Jul 2023 14:32:25 +0100 Subject: [PATCH 076/288] background to int --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index a0d3aade..3079ddd5 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -693,7 +693,7 @@ def __call__( if moment_analysis: self._calculate_temperatures() if background is not None: - self.measured_spectra = self.measured_spectra + background.sel(t=t) + self.measured_spectra = self.measured_spectra + background self._build_bckc_dictionary() return self.bckc From 3690efb9c15d054cca76cb6b59cdb49ce6b80a12 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 11 Jul 2023 11:13:50 +0100 Subject: [PATCH 077/288] stash --- indica/workflows/abstract_bayes_workflow.py | 82 ++++++++++++++------- indica/workflows/bayes_workflow_dev.py | 18 +++-- indica/writers/bda_tree.py | 33 +++++---- 3 files changed, 86 insertions(+), 47 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 1faae4e6..93265f00 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -64,7 +64,7 @@ def setup_models(self, diagnostics: list): self.models = {} @abstractmethod - def setup_optimiser(self): + def setup_optimiser(self, model_kwargs): """ Initialise and provide settings for optimiser """ @@ -93,32 +93,45 @@ def _build_result_dict(self): """ - result = {} result["TIME"] = self.plasma.t result["TIME_OPT"] = self.plasma.time_to_calculate + + quant_list = [item.split(".") for item in self.opt_quantity] + result["MODEL_DATA"] = {diag_name.upper(): + {quantity[1].upper(): self.blobs[f"{quantity[0]}.{quantity[1]}"] + for quantity in quant_list if quantity[0] == diag_name} + for diag_name in self.diagnostics + } + result["MODEL_DATA"]["SAMPLES"] = self.samples + + result["DIAG_DATA"] = {diag_name.upper(): + {quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] + for quantity in quant_list if quantity[0] == diag_name} + for diag_name in self.diagnostics + } + result["INPUT"] = { "BURN_FRAC": self.burn_frac, "ITER": self.iterations, - "MODEL_KWARGS": "KWARGS", - "OPT_KWARGS": "KWARGS", + "MODEL_KWARGS": self.model_kwargs, "NWALKERS": self.nwalkers, "PARAM_NAMES": self.param_names, "PULSE": self.pulse, + "DT": self.dt, } - result["GLOBAL"] = { - "TI0": 0, - "TE0": 0, - "NE0": 0, - "NI0": 0, - "TI0_ERR": 0, - "TE0_ERR": 0, - "NE0_ERR": 0, - "NI0_ERR": 0, + result["INPUT"]["WORKFLOW"] = { + diag_name.upper(): + {"PULSE":self.pulse, # Change this if different pulses used for diagnostics + "USAGE": [quantity[1] for quantity in quant_list if quantity[0] == diag_name].join(), + "RUN": "PLACEHOLDER", + } + for diag_name in self.diagnostics } + result["PHANTOMS"] = { "FLAG": self.phantoms, "NE": self.phantom_profiles["electron_density"], @@ -131,6 +144,25 @@ def _build_result_dict(self): "NIMP2": self.phantom_profiles["impurity_density"].sel(element="c") } + result["PROFILES"] = { + "RHO_POLOIDAL": self.plasma.rho, + "RHO_TOR":self.plasma.equilibrium.rhotor, + + "NE": self.blobs["electron_density"].mean(dim="index"), + "NE_ERR": self.blobs["electron_density"].std(dim="index"), + + "NI": self.blobs["ion_density"].mean(dim="index"), + "TE": self.blobs["electron_temperature"].mean(dim="index"), + "TI": self.blobs["ion_temperature"].mean(dim="index"), + + + "NFAST": self.blobs["fast_density"], + "NNEUTR": self.blobs["neutral_density"], + "NIZ1": self.blobs["impurity_density"].sel(element="ar"), # TODO: generalise + "NIZ2": self.blobs["impurity_density"].sel(element="c") + } + + result["PROFILE_STAT"] = { "SAMPLES": self.samples, "RHO_POLOIDAL": self.plasma.rho, @@ -152,19 +184,19 @@ def _build_result_dict(self): "AUTOCORR": self.autocorr, } - quant_list = [item.split(".") for item in self.opt_quantity] - result["MODEL_DATA"] = {diag_name.upper(): - {quantity[1].upper(): self.blobs[f"{quantity[0]}.{quantity[1]}"] - for quantity in quant_list if quantity[0] == diag_name} - for diag_name in self.diagnostics - } - result["MODEL_DATA"]["SAMPLES"] = self.samples - result["DIAG_DATA"] = {diag_name.upper(): - {quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] - for quantity in quant_list if quantity[0] == diag_name} - for diag_name in self.diagnostics - } + + result["GLOBAL"] = { + "TI0": 0, + "TE0": 0, + "NE0": 0, + "NI0": 0, + "TI0_ERR": 0, + "TE0_ERR": 0, + "NE0_ERR": 0, + "NI0_ERR": 0, + } + self.result = result return self.result diff --git a/indica/workflows/bayes_workflow_dev.py b/indica/workflows/bayes_workflow_dev.py index d82eb87d..b07e21b9 100644 --- a/indica/workflows/bayes_workflow_dev.py +++ b/indica/workflows/bayes_workflow_dev.py @@ -49,7 +49,7 @@ "Ne_prof.wcenter": get_uniform(0.1, 0.8), "Ne_prof.peaking": get_uniform(1, 6), "ar_conc": loguniform(0.0001, 0.01), - "Nimp_prof.y0": get_uniform(1e16, 1e18), + "Nimp_prof.y0": loguniform(1e16, 1e18), "Nimp_prof.y1": get_uniform(1e15, 2e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 @@ -95,7 +95,7 @@ ] OPTIMISED_QUANTITY = [ - # "xrcs.spectra", + "xrcs.spectra", "cxff_pi.ti", "efit.wp", "smmh1.ne"] @@ -106,10 +106,13 @@ def __init__( self, pulse=None, pulse_to_write=None, + diagnostics=None, param_names=None, opt_quantity=None, priors = None, + phantoms=False, phantom_params=None, + model_kwargs = None, nwalkers=50, tstart=0.02, @@ -119,10 +122,8 @@ def __init__( iterations=100, burn_frac=0, - phantoms=False, - diagnostics=None, sample_high_density = False, - mds_write=False, + mds_write = False, fast_particles = False, ): self.pulse = pulse @@ -131,6 +132,7 @@ def __init__( self.opt_quantity = opt_quantity self.priors = priors self.phantom_params = phantom_params + self.model_kwargs = model_kwargs self.tstart = tstart self.tend = tend @@ -237,7 +239,7 @@ def setup_models(self, diagnostics: list): window_vector=self.data[diag][ "spectra" ].wavelength.values - * 0.1, + , ) model.set_los_transform(los_transform) @@ -274,6 +276,7 @@ def setup_optimiser(self, ): log_prob_fn=self.bayesopt.ln_posterior, parameter_names=self.param_names, moves=self.move, + kwargs=self.model_kwargs, ) if self.sample_high_density: @@ -376,7 +379,7 @@ def __call__(self, filepath = "./results/test/", **kwargs): run = DevBayesWorkflow( pulse=10009, - iterations=10, + iterations=5, nwalkers=20, burn_frac=0.10, dt=0.005, @@ -390,6 +393,7 @@ def __call__(self, filepath = "./results/test/", **kwargs): param_names=OPTIMISED_PARAMS, phantom_params=DEFAULT_PHANTOM_PARAMS, priors=DEFAULT_PRIORS, + model_kwargs= {"moment_analysis":False, "background": 100} ) results = run(filepath="./results/test/",) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 66126ee2..1851f998 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -9,6 +9,21 @@ def bda(): nodes = { "TIME": ("NUMERIC", "time vector, s"), "TIME_OPT": ("NUMERIC", "time of optimisation, s"), + + "INPUT": { + "BURN_FRAC": ("NUMERIC", "Burn in fraction for chains"), + "ITER": ("NUMERIC", "Iterations of optimiser"), + "PARAM_NAMES": ("TEXT", "Names of params optimised"), + "MODEL_KWARGS": ("TEXT", "Model key word arguments"), + # "OPT_KWARGS": ("TEXT", "Optimiser key word arguments"), + "PULSE": ("NUMERIC", "Pulse number"), + + "TSTART": ("NUMERIC", "Start of time vector"), + "TEND": ("NUMERIC", "End of time vector"), + "DT": ("NUMERIC", "Distance between time points"), + "TSAMPLE": ("NUMERIC", "Sample time"), + }, + "METADATA": { "GITCOMMIT": ("TEXT", "Commit ID used for run"), "USER": ("TEXT", "Username of owner"), @@ -21,6 +36,7 @@ def bda(): }, "PROFILES": { "RHO_POLOIDAL": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "RHO_TOR": ("NUMERIC", "Radial vector, toroidal flux"), "NE": ("SIGNAL", "Electron density, m^-3"), "NI": ("SIGNAL", "Ion density, m^-3"), "TE": ("SIGNAL", "Electron temperature, eV"), @@ -30,7 +46,6 @@ def bda(): "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), - "TIZ1": ("SIGNAL", "Ion temperature of impurity IMP1, eV"), "TIZ2": ("SIGNAL", "Ion temperature of impurity IMP2, eV"), "TIZ3": ("SIGNAL", "Ion temperature of impurity IMP3, eV"), @@ -94,15 +109,6 @@ def bda(): }, - "INPUT": { - "PARAM_NAMES": ("TEXT", "Names of params optimised"), - "MODEL_KWARGS": ("TEXT", "Model key word arguments"), - "OPT_KWARGS": ("TEXT", "Optimiser key word arguments"), - "ITER": ("NUMERIC", "Iterations of optimiser"), - "NWALKERS": ("NUMERIC", "Number of walkers used"), - "BURN_FRAC": ("NUMERIC", "Burn in fraction for chains"), - }, - "OPTIMISATION": { "AUTO_CORR": ("NUMERIC", "Auto-correlation (iteration nwalker)"), "POST_SAMPLE": ("NUMERIC", "Posterior probability samples (sample)"), @@ -120,7 +126,6 @@ def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_qua quant_list = [item.split(".") for item in diagnostic_quantities] # replace OPTIMISED_QUANTITY diag_names = list(set([item[0] for item in quant_list])) - diag_nodes = {diag_name: {quantity[1]: ("SIGNAL", f"measured {quantity[1]} from {quantity[0]}") for quantity in quant_list if quantity[0] == diag_name} @@ -129,7 +134,6 @@ def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_qua for diag_name in diag_names: diag_nodes[diag_name]["RUN"] = ("TEXT", f"RUN from which {diag_name} data was taken") - nodes = { "RUN": ("TEXT", "RUN used for diagnostic"), "USAGE": ("TEXT", "Diagnostic used in analysis"), @@ -137,9 +141,8 @@ def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_qua } workflow_nodes = {diag_name: nodes - for diag_name in diag_names - } - + for diag_name in diag_names + } model_nodes = {diag_name: {quantity[1]: ("SIGNAL", f"modelled {quantity[1]} from {quantity[0]}") From 3c23661233f5b196d1f7c1b6ab259bcf68e567c2 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 11 Jul 2023 13:52:15 +0100 Subject: [PATCH 078/288] renamed impurities --- indica/workflows/bayes_plots.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 9d781436..dc9ed77d 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -403,7 +403,7 @@ def plot_bayes_result( color="green", ) - key = "NIMP1" + key = "NIZ1" plot_profile( profiles[key], key, @@ -414,7 +414,7 @@ def plot_bayes_result( color="red", ) - key = "NIMP2" + key = "NIZ2" plot_profile( profiles[key], key, From bf3bfb9f1437fb8b9812d5f107c735fe5d65ff06 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 12 Jul 2023 11:58:24 +0100 Subject: [PATCH 079/288] stashing --- indica/workflows/abstract_bayes_workflow.py | 162 ++++++++++++++------ indica/workflows/bayes_workflow_dev.py | 64 ++++---- indica/writers/bda_tree.py | 98 ++++++++---- 3 files changed, 216 insertions(+), 108 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 93265f00..06f237b0 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -88,114 +88,164 @@ def save_phantom_profiles(self, kinetic_profiles=None): def _build_result_dict(self): """ - TODO: xarray to numpy for MDSPlus writing (plotting / dimensions / units have to be figured out) + Returns + ------- + + dictionary of results in MDS+ structure """ result = {} + quant_list = [item.split(".") for item in self.opt_quantity] + result["TIME"] = self.plasma.t result["TIME_OPT"] = self.plasma.time_to_calculate - - quant_list = [item.split(".") for item in self.opt_quantity] - result["MODEL_DATA"] = {diag_name.upper(): - {quantity[1].upper(): self.blobs[f"{quantity[0]}.{quantity[1]}"] - for quantity in quant_list if quantity[0] == diag_name} - for diag_name in self.diagnostics - } - result["MODEL_DATA"]["SAMPLES"] = self.samples - - result["DIAG_DATA"] = {diag_name.upper(): - {quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] - for quantity in quant_list if quantity[0] == diag_name} - for diag_name in self.diagnostics - } + result["METADATA"] = { + "GITCOMMIT": "PLACEHOLDER", + "USER": "PLACEHOLDER", + "EQUIL": "PLACEHOLDER", + } result["INPUT"] = { "BURN_FRAC": self.burn_frac, "ITER": self.iterations, - "MODEL_KWARGS": self.model_kwargs, "NWALKERS": self.nwalkers, + + "MODEL_KWARGS": self.model_kwargs, + "OPT_QUANTITY": self.opt_quantity, "PARAM_NAMES": self.param_names, + "PULSE": self.pulse, "DT": self.dt, + "IMPURITIES":self.plasma.impurities, + "MAIN_ION": self.plasma.main_ion, } - result["INPUT"]["WORKFLOW"] = { diag_name.upper(): - {"PULSE":self.pulse, # Change this if different pulses used for diagnostics - "USAGE": [quantity[1] for quantity in quant_list if quantity[0] == diag_name].join(), + {"PULSE": self.pulse, # Change this if different pulses used for diagnostics + "USAGE": "".join([quantity[1] for quantity in quant_list if quantity[0] == diag_name]), "RUN": "PLACEHOLDER", } for diag_name in self.diagnostics } + result["MODEL_DATA"] = {diag_name.upper(): + {quantity[1].upper(): self.blobs[f"{quantity[0]}.{quantity[1]}"] + for quantity in quant_list if quantity[0] == diag_name} + for diag_name in self.diagnostics + } + result["MODEL_DATA"]["SAMPLES"] = self.samples + + result["DIAG_DATA"] = {diag_name.upper(): + {quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] + for quantity in quant_list if quantity[0] == diag_name} + for diag_name in self.diagnostics + } + result["PHANTOMS"] = { "FLAG": self.phantoms, "NE": self.phantom_profiles["electron_density"], "TE": self.phantom_profiles["electron_temperature"], - "TI": self.phantom_profiles["ion_temperature"], - "NI": self.phantom_profiles["ion_density"], + "TI": self.phantom_profiles["ion_temperature"].sel(element=self.plasma.main_ion), + "NI": self.phantom_profiles["ion_density"].sel(element=self.plasma.main_ion), "NNEUTR": self.phantom_profiles["neutral_density"], "NFAST": self.phantom_profiles["fast_density"], - "NIMP1": self.phantom_profiles["impurity_density"].sel(element="ar"), # TODO: generalise - "NIMP2": self.phantom_profiles["impurity_density"].sel(element="c") } + result["PHANTOMS"] = {**result["PHANTOMS"], **{ + f"NIZ{num_imp+1}": self.phantom_profiles["impurity_density"].sel(element=imp) + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["PHANTOMS"] = {**result["PHANTOMS"], **{ + f"TIZ{num_imp + 1}": self.phantom_profiles["ion_temperature"].sel(element=imp) + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["PROFILES"] = { "RHO_POLOIDAL": self.plasma.rho, - "RHO_TOR":self.plasma.equilibrium.rhotor, - - "NE": self.blobs["electron_density"].mean(dim="index"), - "NE_ERR": self.blobs["electron_density"].std(dim="index"), + "RHO_TOR": self.plasma.equilibrium.rhotor.interp(t=self.plasma.t), - "NI": self.blobs["ion_density"].mean(dim="index"), - "TE": self.blobs["electron_temperature"].mean(dim="index"), - "TI": self.blobs["ion_temperature"].mean(dim="index"), + "NE": self.blobs["electron_density"].median(dim="index"), + "NI": self.blobs["ion_density"].sel(element=self.plasma.main_ion).median(dim="index"), + "TE": self.blobs["electron_temperature"].median(dim="index"), + "TI": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion).median(dim="index"), + "NFAST": self.blobs["fast_density"].median(dim="index"), + "NNEUTR": self.blobs["neutral_density"].median(dim="index"), + "NE_ERR": self.blobs["electron_density"].std(dim="index"), + "NI_ERR": self.blobs["ion_density"].sel(element=self.plasma.main_ion).std(dim="index"), + "TE_ERR": self.blobs["electron_temperature"].std(dim="index"), + "TI_ERR": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion).std(dim="index"), + "NFAST_ERR": self.blobs["fast_density"].std(dim="index"), + "NNEUTR_ERR": self.blobs["neutral_density"].std(dim="index"), - "NFAST": self.blobs["fast_density"], - "NNEUTR": self.blobs["neutral_density"], - "NIZ1": self.blobs["impurity_density"].sel(element="ar"), # TODO: generalise - "NIZ2": self.blobs["impurity_density"].sel(element="c") } + result["PROFILES"] = {**result["PROFILES"], **{ + f"NIZ{num_imp+1}": self.blobs["impurity_density"].sel(element=imp).median(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["PROFILES"] = {**result["PROFILES"], **{ + f"NIZ{num_imp+1}_ERR": self.blobs["impurity_density"].sel(element=imp).std(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["PROFILES"] = {**result["PROFILES"], **{ + f"TIZ{num_imp + 1}": self.blobs["ion_temperature"].sel(element=imp).median(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["PROFILES"] = {**result["PROFILES"], **{ + f"TIZ{num_imp + 1}_ERR": self.blobs["ion_temperature"].sel(element=imp).std(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities)}} result["PROFILE_STAT"] = { "SAMPLES": self.samples, "RHO_POLOIDAL": self.plasma.rho, "NE": self.blobs["electron_density"], - "NI": self.blobs["ion_density"], + "NI": self.blobs["ion_density"].sel(element=self.plasma.main_ion), "TE": self.blobs["electron_temperature"], - "TI": self.blobs["ion_temperature"], + "TI": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion), "NFAST": self.blobs["fast_density"], "NNEUTR": self.blobs["neutral_density"], - "NIMP1": self.blobs["impurity_density"].sel(element="ar"), # TODO: generalise - "NIMP2": self.blobs["impurity_density"].sel(element="c") } + result["PROFILE_STAT"] = {**result["PROFILE_STAT"], **{ + f"NIZ{num_imp+1}": self.blobs["impurity_density"].sel(element=imp) + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["PROFILE_STAT"] = {**result["PROFILE_STAT"], **{ + f"TIZ{num_imp + 1}": self.blobs["ion_temperature"].sel(element=imp) + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["OPTIMISATION"] = { - "OPT_QUANTITY": self.opt_quantity, "ACCEPT_FRAC": self.accept_frac, "PRIOR_SAMPLE": self.prior_sample, "POST_SAMPLE": self.post_sample, "AUTOCORR": self.autocorr, } - - result["GLOBAL"] = { - "TI0": 0, - "TE0": 0, - "NE0": 0, - "NI0": 0, - "TI0_ERR": 0, - "TE0_ERR": 0, - "NE0_ERR": 0, - "NI0_ERR": 0, + "TI0": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion).sel(rho_poloidal=0, method="nearest").median(dim="index"), + "TE0": self.blobs["electron_temperature"].sel(rho_poloidal=0, method="nearest").median(dim="index"), + "NE0": self.blobs["electron_density"].sel(rho_poloidal=0, method="nearest").median(dim="index"), + "NI0": self.blobs["ion_density"].sel(element=self.plasma.main_ion).sel(rho_poloidal=0, method="nearest").median(dim="index"), + "TI0_ERR": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion).sel(rho_poloidal=0, method="nearest").std(dim="index"), + "TE0_ERR": self.blobs["electron_temperature"].sel(rho_poloidal=0, method="nearest").std(dim="index"), + "NE0_ERR": self.blobs["electron_density"].sel(rho_poloidal=0, method="nearest").std(dim="index"), + "NI0_ERR": self.blobs["ion_density"].sel(element=self.plasma.main_ion).sel(rho_poloidal=0, method="nearest").std(dim="index"), } + result["GLOBAL"] = {**result["GLOBAL"], **{ + f"TI0Z{num_imp + 1}": self.blobs["ion_temperature"].sel(element=imp).sel(rho_poloidal=0, method="nearest").median(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["GLOBAL"] = {**result["GLOBAL"], **{ + f"TI0Z{num_imp + 1}_ERR": self.blobs["ion_temperature"].sel(element=imp).sel(rho_poloidal=0, + method="nearest").std( + dim="index") + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["GLOBAL"] = {**result["GLOBAL"], **{ + f"NI0Z{num_imp + 1}": self.blobs["impurity_density"].sel(element=imp).sel(rho_poloidal=0, method="nearest").median(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities)}} + result["GLOBAL"] = {**result["GLOBAL"], **{ + f"NI0Z{num_imp + 1}_ERR": self.blobs["impurity_density"].sel(element=imp).sel(rho_poloidal=0, + method="nearest").std( + dim="index") + for num_imp, imp in enumerate(self.plasma.impurities)}} self.result = result return self.result @@ -242,10 +292,24 @@ def save_pickle(self, filepath): with open(filepath + "results.pkl", "wb") as handle: pickle.dump(self.result, handle) + def dict_of_dataarray_to_numpy(self, dict_of_dataarray): + """ + Mutates input dictionary to change xr.DataArray objects to np.array + + """ + for key, value in dict_of_dataarray.items(): + if isinstance(value, dict): + self.dict_of_dataarray_to_numpy(value) + elif isinstance(value, xr.DataArray): + dict_of_dataarray[key] = dict_of_dataarray[key].values + return dict_of_dataarray + @abstractmethod def __call__(self, filepath="./results/test/", **kwargs): self.run_sampler() self.save_pickle(filepath=filepath) + self.result = self.dict_of_dataarray_to_numpy(self.result) + return self.result diff --git a/indica/workflows/bayes_workflow_dev.py b/indica/workflows/bayes_workflow_dev.py index b07e21b9..ba0e97ed 100644 --- a/indica/workflows/bayes_workflow_dev.py +++ b/indica/workflows/bayes_workflow_dev.py @@ -1,6 +1,7 @@ import emcee import numpy as np import flatdict +import copy from scipy.stats import loguniform from indica.readers.read_st40 import ReadST40 @@ -122,17 +123,20 @@ def __init__( iterations=100, burn_frac=0, + mds_write=False, + plot=True, sample_high_density = False, - mds_write = False, fast_particles = False, ): self.pulse = pulse self.pulse_to_write = pulse_to_write + self.diagnostics = diagnostics self.param_names = param_names self.opt_quantity = opt_quantity self.priors = priors self.phantom_params = phantom_params self.model_kwargs = model_kwargs + self.phantoms = phantoms self.tstart = tstart self.tend = tend @@ -142,10 +146,9 @@ def __init__( self.iterations = iterations self.burn_frac = burn_frac - self.phantoms = phantoms - self.diagnostics = diagnostics - self.sample_high_density = sample_high_density self.mds_write = mds_write + self.plot = plot + self.sample_high_density = sample_high_density self.fast_particles = fast_particles for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics", "phantom_params"]: @@ -154,10 +157,10 @@ def __init__( self.setup_plasma() self.save_phantom_profiles() - self.read_data(diagnostics) + self.read_data(self.diagnostics) self.setup_opt_data(phantoms=self.phantoms) - self.setup_models(diagnostics) - self.setup_optimiser() + self.setup_models(self.diagnostics) + self.setup_optimiser(self.model_kwargs) def setup_plasma(self): self.plasma = Plasma( @@ -258,7 +261,7 @@ def setup_models(self, diagnostics: list): self.models[diag] = model - def setup_optimiser(self, ): + def setup_optimiser(self, model_kwargs): self.bayesopt = BayesModels( plasma=self.plasma, @@ -276,7 +279,7 @@ def setup_optimiser(self, ): log_prob_fn=self.bayesopt.ln_posterior, parameter_names=self.param_names, moves=self.move, - kwargs=self.model_kwargs, + kwargs=model_kwargs, ) if self.sample_high_density: @@ -364,38 +367,47 @@ def _exp_data(self): ) def __call__(self, filepath = "./results/test/", **kwargs): + + if self.mds_write: + self.node_structure = create_nodes(pulse_to_write=self.pulse_to_write, + diagnostic_quantities=self.opt_quantity, + mode="NEW") + self.run_sampler() self.save_pickle(filepath=filepath) - plot_bayes_result(self.result, filepath) + self.result = self.dict_of_dataarray_to_numpy(self.result) + + if self.plot: + plot_bayes_result(self.result, filepath) + + if self.mds_write: + write_nodes(self.pulse_to_write, self.node_structure, self.result) + return self.result if __name__ == "__main__": - # mds_write=False - # pulse_to_write = 23000101 - # if mds_write: - # node_structure = create_nodes(pulse_to_write=pulse_to_write, diagnostic_quantities=OPTIMISED_QUANTITY) - run = DevBayesWorkflow( pulse=10009, + pulse_to_write=23000101, + diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], + opt_quantity=OPTIMISED_QUANTITY, + param_names=OPTIMISED_PARAMS, + phantom_params=DEFAULT_PHANTOM_PARAMS, + priors=DEFAULT_PRIORS, + iterations=5, nwalkers=20, burn_frac=0.10, dt=0.005, tsample=0.060, - diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], - phantoms=False, - sample_high_density=True, - mds_write=False, - opt_quantity=OPTIMISED_QUANTITY, - param_names=OPTIMISED_PARAMS, - phantom_params=DEFAULT_PHANTOM_PARAMS, - priors=DEFAULT_PRIORS, - model_kwargs= {"moment_analysis":False, "background": 100} + mds_write=True, + plot=False, + phantoms=False, + sample_high_density=False, + model_kwargs= { "background": 100} ) results = run(filepath="./results/test/",) - # if mds_write: - # write_nodes(pulse_to_write, node_structure, run.nested_results) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 1851f998..1d16083c 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -1,4 +1,5 @@ # Create trees from ppac standard utility tools + import standard_utility as util from MDSplus import Connection import sys @@ -13,46 +14,44 @@ def bda(): "INPUT": { "BURN_FRAC": ("NUMERIC", "Burn in fraction for chains"), "ITER": ("NUMERIC", "Iterations of optimiser"), - "PARAM_NAMES": ("TEXT", "Names of params optimised"), + "PARAM_NAMES": ("TEXT", "Names of parameters optimised"), + "OPT_QUANTITY":("TEXT", "Names of quantities optimised"), "MODEL_KWARGS": ("TEXT", "Model key word arguments"), # "OPT_KWARGS": ("TEXT", "Optimiser key word arguments"), "PULSE": ("NUMERIC", "Pulse number"), - "TSTART": ("NUMERIC", "Start of time vector"), - "TEND": ("NUMERIC", "End of time vector"), - "DT": ("NUMERIC", "Distance between time points"), - "TSAMPLE": ("NUMERIC", "Sample time"), + "TSTART": ("NUMERIC", "Start of time vector, s"), + "TEND": ("NUMERIC", "End of time vector, s"), + "DT": ("NUMERIC", "Distance between time points, s"), + "TSAMPLE": ("NUMERIC", "Sample time, s"), + "IMPURITIES": ("TEXT", "Names of impurity elements"), + "MAIN_ION": ("TEXT", "Name of main ion"), + }, "METADATA": { "GITCOMMIT": ("TEXT", "Commit ID used for run"), "USER": ("TEXT", "Username of owner"), - "PULSE": ("NUMERIC", "Pulse number analysed"), "EQUIL": ("TEXT", "Equilibrium used"), - "MAIN_ION": ("TEXT", "Main ion element"), - "IMP1": ("TEXT", "Impurity element chosen for Z1"), - "IMP2": ("TEXT", "Impurity element chosen for Z2"), - "DIAG_RUNS": ("TEXT", "RUNS from which diagnostic data originated") }, + "PROFILES": { - "RHO_POLOIDAL": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), - "RHO_TOR": ("NUMERIC", "Radial vector, toroidal flux"), + "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "RHOT": ("SIGNAL", "Radial vector, toroidal flux"), + "NE": ("SIGNAL", "Electron density, m^-3"), "NI": ("SIGNAL", "Ion density, m^-3"), "TE": ("SIGNAL", "Electron temperature, eV"), "TI": ("SIGNAL", "Ion temperature of main ion, eV"), - "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), - "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), - "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), - "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), - - "TIZ1": ("SIGNAL", "Ion temperature of impurity IMP1, eV"), - "TIZ2": ("SIGNAL", "Ion temperature of impurity IMP2, eV"), - "TIZ3": ("SIGNAL", "Ion temperature of impurity IMP3, eV"), - "NIZ1": ("SIGNAL", "Density of impurity IMP1, m^-3"), - "NIZ2": ("SIGNAL", "Density of impurity IMP2, m^-3"), - "NIZ3": ("SIGNAL", "Density of impurity IMP3, m^-3"), + "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1, eV"), + "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), + "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), + "NIZ1": ("SIGNAL", "Density of impurity Z1, m^-3"), + "NIZ2": ("SIGNAL", "Density of impurity Z2, m^-3"), + "NIZ3": ("SIGNAL", "Density of impurity Z3, m^-3"), "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), + "NFAST": ("SIGNAL", "Density of fast ion, m^-3"), + "ZI": ("SIGNAL", "Average charge of main ion, "), "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), @@ -60,16 +59,29 @@ def bda(): "ZEFF": ("SIGNAL", "Effective charge, "), "P": ("SIGNAL", "Pressure,Pa"), "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), + + "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), + "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), + "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), + "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), + "TIZ1_ERR": ("SIGNAL", "Ion temperature of impurity Z1 error, eV"), + "TIZ2_ERR": ("SIGNAL", "Ion temperature of impurity Z2 error, eV"), + "TIZ3_ERR": ("SIGNAL", "Ion temperature of impurity Z3 error, eV"), + "NIZ1_ERR": ("SIGNAL", "Density of impurity Z1 error, m^-3"), + "NIZ2_ERR": ("SIGNAL", "Density of impurity Z2 error, m^-3"), + "NIZ3_ERR": ("SIGNAL", "Density of impurity Z3 error, m^-3"), + "NNEUTR_ERR": ("SIGNAL", "Density of neutral main ion error, m^-3"), + "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), + }, "PROFILE_STAT": { "SAMPLES": ("NUMERIC", "Numerical index of the optimisation samples"), - "RHO_POLOIDAL": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), "NE": ("SIGNAL", "Electron density, m^-3"), "NI": ("SIGNAL", "Ion density, m^-3"), "TE": ("SIGNAL", "Electron temperature, eV"), "TI": ("SIGNAL", "Ion temperature of main ion, eV"), - "TIZ1": ("SIGNAL", "Ion temperature of impurity IMP1, eV"), "TIZ2": ("SIGNAL", "Ion temperature of impurity IMP2, eV"), "TIZ3": ("SIGNAL", "Ion temperature of impurity IMP3, eV"), @@ -77,13 +89,15 @@ def bda(): "NIZ2": ("SIGNAL", "Density of impurity IMP2, m^-3"), "NIZ3": ("SIGNAL", "Density of impurity IMP3, m^-3"), "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), + "NFAST": ("SIGNAL", "Density of fast ions, m^-3"), + "ZI": ("SIGNAL", "Average charge of main ion, "), "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), "ZEFF": ("SIGNAL", "Effective charge, "), "P": ("SIGNAL", "Pressure,Pa"), - "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), + "VOLUME": ("SIGNAL", "Volume inside magnetic surface, m^3"), }, "GLOBAL": { @@ -91,11 +105,23 @@ def bda(): "NI0": ("SIGNAL", "Central ion density, m^-3"), "TE0": ("SIGNAL", "Central electron temperature, eV"), "TI0": ("SIGNAL", "Central ion temperature of main ion, eV"), + "TI0Z1": ("SIGNAL", "Central ion temperature of impurity Z1, eV"), + "TI0Z2": ("SIGNAL", "Central ion temperature of impurity Z2, eV"), + "TI0Z3": ("SIGNAL", "Central ion temperature of impurity Z3, eV"), + "NI0Z1": ("SIGNAL", "Central density of impurity Z1, m^-3"), + "NI0Z2": ("SIGNAL", "Central density of impurity Z2, m^-3"), + "NI0Z3": ("SIGNAL", "Central density of impurity Z3, m^-3"), "NE0_ERR": ("SIGNAL", "Central electron density error, m^-3"), "NI0_ERR": ("SIGNAL", "Central ion density error, m^-3"), "TE0_ERR": ("SIGNAL", "Central electron temperature error, eV"), "TI0_ERR": ("SIGNAL", "Central ion temperature of main ion error, eV"), + "TI0Z1_ERR": ("SIGNAL", "Central ion temperature of impurity Z1, eV"), + "TI0Z2_ERR": ("SIGNAL", "Central ion temperature of impurity Z2, eV"), + "TI0Z3_ERR": ("SIGNAL", "Central ion temperature of impurity Z3, eV"), + "NI0Z1_ERR": ("SIGNAL", "Central density of impurity Z1, m^-3"), + "NI0Z2_ERR": ("SIGNAL", "Central density of impurity Z2, m^-3"), + "NI0Z3_ERR": ("SIGNAL", "Central density of impurity Z3, m^-3"), }, "PHANTOMS": { @@ -105,11 +131,17 @@ def bda(): "NI": ("SIGNAL", "Ion density, m^-3"), "TE": ("SIGNAL", "Electron temperature, eV"), "TI": ("SIGNAL", "Ion temperature of main ion, eV"), - # "NIMP1": ("SIGNAL", "Impurity density, m^-3"), + "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1 , eV"), + "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), + "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), + "NIZ1": ("SIGNAL", "Impurity density of Z1, m^-3"), + "NIZ2": ("SIGNAL", "Impurity density of Z2, m^-3"), + "NIZ3": ("SIGNAL", "Impurity density of Z3, m^-3"), }, "OPTIMISATION": { + "ACCEPT_FRAC": ("NUMERIC", "Fraction of samples accepted by optimiser"), "AUTO_CORR": ("NUMERIC", "Auto-correlation (iteration nwalker)"), "POST_SAMPLE": ("NUMERIC", "Posterior probability samples (sample)"), "PRIOR_SAMPLE": ("NUMERIC", "Prior samples"), @@ -121,9 +153,9 @@ def bda(): DIAGNOSTIC_QUANTITY = ["DIAGNOSTIC1.QUANTITY1", "DIAGNOSTIC1.QUANTITY2", "DIAGNOSTIC2.QUANTITY1"] -def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_quantities=DIAGNOSTIC_QUANTITY): +def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_quantities=DIAGNOSTIC_QUANTITY, mode="EDIT"): bda_nodes = bda() - quant_list = [item.split(".") for item in diagnostic_quantities] # replace OPTIMISED_QUANTITY + quant_list = [item.upper().split(".") for item in diagnostic_quantities] # replace OPTIMISED_QUANTITY diag_names = list(set([item[0] for item in quant_list])) diag_nodes = {diag_name: @@ -131,12 +163,12 @@ def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_qua for quantity in quant_list if quantity[0] == diag_name} for diag_name in diag_names } - for diag_name in diag_names: - diag_nodes[diag_name]["RUN"] = ("TEXT", f"RUN from which {diag_name} data was taken") + # for diag_name in diag_names: + # diag_nodes[diag_name]["RUN"] = ("TEXT", f"RUN from which {diag_name} data was taken") nodes = { "RUN": ("TEXT", "RUN used for diagnostic"), - "USAGE": ("TEXT", "Diagnostic used in analysis"), + "USAGE": ("TEXT", "Quantity used in analysis"), "PULSE": ("NUMERIC", "Pulse used for diagnostic") } @@ -175,7 +207,7 @@ def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_qua util.StandardNodeCreation( pulse_number=pulse_to_write, dict_node_info=node_info, - mode="EDIT", + mode=mode, name_of_BEST="BEST", # name of the structure linked to BEST link_BEST_to_run=best, ) From fbfafe5523649bf4a380bb4750a8b0e63ab38aca Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 13 Jul 2023 15:17:41 +0100 Subject: [PATCH 080/288] stashing --- indica/workflows/abstract_bayes_workflow.py | 11 ++++++----- indica/workflows/bayes_workflow_dev.py | 6 +++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 06f237b0..86d1fb99 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -154,13 +154,14 @@ def _build_result_dict(self): "NNEUTR": self.phantom_profiles["neutral_density"], "NFAST": self.phantom_profiles["fast_density"], } - result["PHANTOMS"] = {**result["PHANTOMS"], **{ + result["PHANTOMS"].update({ f"NIZ{num_imp+1}": self.phantom_profiles["impurity_density"].sel(element=imp) - for num_imp, imp in enumerate(self.plasma.impurities)}} - result["PHANTOMS"] = {**result["PHANTOMS"], **{ + for num_imp, imp in enumerate(self.plasma.impurities) + }) + result["PHANTOMS"].update({ f"TIZ{num_imp + 1}": self.phantom_profiles["ion_temperature"].sel(element=imp) - for num_imp, imp in enumerate(self.plasma.impurities)}} - + for num_imp, imp in enumerate(self.plasma.impurities) + }) result["PROFILES"] = { "RHO_POLOIDAL": self.plasma.rho, diff --git a/indica/workflows/bayes_workflow_dev.py b/indica/workflows/bayes_workflow_dev.py index ba0e97ed..5fa34615 100644 --- a/indica/workflows/bayes_workflow_dev.py +++ b/indica/workflows/bayes_workflow_dev.py @@ -15,7 +15,7 @@ from indica.converters.line_of_sight import LineOfSightTransform from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow -from indica.writers.bda_tree import create_nodes, write_nodes +from indica.writers.bda_tree import create_nodes, write_nodes, check_analysis_run # global configurations DEFAULT_PHANTOM_PARAMS = { @@ -107,6 +107,7 @@ def __init__( self, pulse=None, pulse_to_write=None, + run="RUN01", diagnostics=None, param_names=None, opt_quantity=None, @@ -130,6 +131,7 @@ def __init__( ): self.pulse = pulse self.pulse_to_write = pulse_to_write + self.run = run self.diagnostics = diagnostics self.param_names = param_names self.opt_quantity = opt_quantity @@ -369,6 +371,7 @@ def _exp_data(self): def __call__(self, filepath = "./results/test/", **kwargs): if self.mds_write: + check_analysis_run(self.pulse, self.run) self.node_structure = create_nodes(pulse_to_write=self.pulse_to_write, diagnostic_quantities=self.opt_quantity, mode="NEW") @@ -391,6 +394,7 @@ def __call__(self, filepath = "./results/test/", **kwargs): run = DevBayesWorkflow( pulse=10009, pulse_to_write=23000101, + run="RUN01", diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], opt_quantity=OPTIMISED_QUANTITY, param_names=OPTIMISED_PARAMS, From 9f62a78bde2e100f5a8e5f7a20acb5c9dcb7f2a5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 14 Jul 2023 12:46:55 +0100 Subject: [PATCH 081/288] stashing --- indica/workflows/bayes_workflow_example.py | 24 +++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index ea954787..4093c5c0 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -345,17 +345,31 @@ def __call__(self, filepath = "./results/test/", **kwargs): if __name__ == "__main__": run = ExampleWorkflow( pulse=10009, + pulse_to_write=None, dt=0.005, tsample=0.060, - diagnostics=["efit", "smmh1", "cxff_pi"], - param_names=OPTIMISED_PARAMS, - opt_quantity=OPTIMISED_QUANTITY, - priors=DEFAULT_PRIORS, + diagnostics=None, + param_names=None, + opt_quantity=None, + priors=None, + phantoms=False, + phantom_params=None, + model_kwargs=None, iterations=20, nwalkers=50, burn_frac=0.10, - sample_high_density=True + tstart=0.02, + tend=0.10, + dt=0.01, + tsample=0.06, + + sample_high_density=True, + + + mds_write=False, + plot=True, + sample_high_density=False, ) test = run(filepath="./results/test/") From 3828c49f4058bf09610c99304049af576d0c106c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 14 Jul 2023 13:31:32 +0100 Subject: [PATCH 082/288] fixing key names --- indica/workflows/bayes_plots.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index dc9ed77d..a4e06773 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -292,7 +292,7 @@ def plot_bayes_result( figheader=figheader, ylabel="Energy [J]", ) - key = "smmh1.ne" + key = "SMMH1.NE" if key in model_data.keys(): violinplot( model_data, @@ -362,7 +362,7 @@ def plot_bayes_result( ) key = "TI" plot_profile( - profiles[key].sel(element="ar"), + profiles[key], key, figheader=figheader, filename="temperature", @@ -384,7 +384,7 @@ def plot_bayes_result( ) key = "NI" plot_profile( - profiles[key].sel(element="h"), + profiles[key], key, figheader=figheader, filetype=filetype, From 310a440c9ffb36f2c55355edbf5f5687e3afa046 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 14 Jul 2023 13:31:58 +0100 Subject: [PATCH 083/288] updating and testing example --- indica/workflows/bayes_workflow_example.py | 124 ++++++++++++--------- 1 file changed, 69 insertions(+), 55 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 4093c5c0..da6cbb22 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -1,15 +1,12 @@ -import pickle - import emcee import numpy as np -import xarray as xr -import pandas as pd import flatdict +import copy from scipy.stats import loguniform -from pathlib import Path from indica.readers.read_st40 import ReadST40 from indica.bayesmodels import BayesModels, get_uniform +from indica.workflows.bayes_plots import plot_bayes_result from indica.models.interferometry import Interferometry from indica.models.helike_spectroscopy import Helike_spectroscopy from indica.models.charge_exchange import ChargeExchange @@ -17,11 +14,10 @@ from indica.models.plasma import Plasma from indica.converters.line_of_sight import LineOfSightTransform -from abstract_bayes_workflow import AbstractBayesWorkflow +from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow +from indica.writers.bda_tree import create_nodes, write_nodes, check_analysis_run # global configurations -from indica.workflows.bayes_plots import plot_bayes_result - DEFAULT_PHANTOM_PARAMS = { "Ne_prof.y0": 5e19, "Ne_prof.wcenter": 0.4, @@ -54,7 +50,7 @@ "Ne_prof.wcenter": get_uniform(0.1, 0.8), "Ne_prof.peaking": get_uniform(1, 6), "ar_conc": loguniform(0.0001, 0.01), - "Nimp_prof.y0": get_uniform(1e16, 1e18), + "Nimp_prof.y0": loguniform(1e16, 1e18), "Nimp_prof.y1": get_uniform(1e15, 2e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 @@ -99,17 +95,26 @@ # "Ti_prof.peaking", ] -OPTIMISED_QUANTITY = ["cxff_pi.ti", "efit.wp", "smmh1.ne"] +OPTIMISED_QUANTITY = [ + "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne"] -class ExampleWorkflow(AbstractBayesWorkflow): +class DevBayesWorkflow(AbstractBayesWorkflow): def __init__( self, pulse=None, + pulse_to_write=None, + run="RUN01", + diagnostics=None, param_names=None, opt_quantity=None, priors = None, - diagnostics=None, + phantoms=False, + phantom_params=None, + model_kwargs = None, nwalkers=50, tstart=0.02, @@ -119,37 +124,43 @@ def __init__( iterations=100, burn_frac=0, - phantoms=False, + mds_write=False, + plot=True, sample_high_density = False, ): - self.pulse = pulse + self.pulse_to_write = pulse_to_write + self.run = run + self.diagnostics = diagnostics self.param_names = param_names self.opt_quantity = opt_quantity self.priors = priors - self.diagnostics = diagnostics + self.phantom_params = phantom_params + self.model_kwargs = model_kwargs + self.phantoms = phantoms self.tstart = tstart self.tend = tend self.dt = dt self.tsample = tsample - self.nwalkers = nwalkers self.iterations = iterations self.burn_frac = burn_frac - self.phantoms = phantoms + + self.mds_write = mds_write + self.plot = plot self.sample_high_density = sample_high_density - for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics"]: + for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics", "phantom_params"]: if getattr(self, attribute) is None: raise ValueError(f"{attribute} needs to be defined") self.setup_plasma() self.save_phantom_profiles() - self.read_data(diagnostics) + self.read_data(self.diagnostics) self.setup_opt_data(phantoms=self.phantoms) - self.setup_models(diagnostics) - self.setup_optimiser() + self.setup_models(self.diagnostics) + self.setup_optimiser(self.model_kwargs) def setup_plasma(self): self.plasma = Plasma( @@ -168,7 +179,7 @@ def setup_plasma(self): self.plasma.time_to_calculate = self.plasma.t[ np.abs(self.tsample - self.plasma.t).argmin() ] - self.plasma.update_profiles(DEFAULT_PHANTOM_PARAMS) + self.plasma.update_profiles(self.phantom_params) self.plasma.build_atomic_data(calc_power_loss=False) def setup_opt_data(self, phantoms=False): @@ -200,7 +211,6 @@ def setup_models(self, diagnostics: list): los_transform.set_equilibrium(self.plasma.equilibrium) model = Interferometry(name=diag) model.set_los_transform(los_transform) - model.plasma = self.plasma elif diag == "xrcs": los_transform = self.data["xrcs"]["te_kw"].transform @@ -213,24 +223,23 @@ def setup_models(self, diagnostics: list): , ) model.set_los_transform(los_transform) - model.plasma = self.plasma elif diag == "efit": model = EquilibriumReconstruction(name="efit") - model.plasma = self.plasma elif diag == "cxff_pi": transform = self.data[diag]["ti"].transform transform.set_equilibrium(self.plasma.equilibrium) model = ChargeExchange(name=diag, element="ar") model.set_transect_transform(transform) - model.plasma = self.plasma else: raise ValueError(f"{diag} not found in setup_models") + model.plasma = self.plasma + self.models[diag] = model - def setup_optimiser(self, ): + def setup_optimiser(self, model_kwargs): self.bayesopt = BayesModels( plasma=self.plasma, @@ -248,6 +257,7 @@ def setup_optimiser(self, ): log_prob_fn=self.bayesopt.ln_posterior, parameter_names=self.param_names, moves=self.move, + kwargs=model_kwargs, ) if self.sample_high_density: @@ -335,45 +345,49 @@ def _exp_data(self): ) def __call__(self, filepath = "./results/test/", **kwargs): + + if self.mds_write: + # check_analysis_run(self.pulse, self.run) + self.node_structure = create_nodes(pulse_to_write=self.pulse_to_write, + diagnostic_quantities=self.opt_quantity, + mode="NEW") + self.run_sampler() self.save_pickle(filepath=filepath) - plot_bayes_result(self.result, filepath) - return self.result + if self.plot: # currently requires result + plot_bayes_result(self.result, filepath) + + self.result = self.dict_of_dataarray_to_numpy(self.result) + if self.mds_write: + write_nodes(self.pulse_to_write, self.node_structure, self.result) + + return self.result if __name__ == "__main__": - run = ExampleWorkflow( + + run = DevBayesWorkflow( pulse=10009, - pulse_to_write=None, + pulse_to_write=23000101, + run="RUN01", + diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], + opt_quantity=OPTIMISED_QUANTITY, + param_names=OPTIMISED_PARAMS, + phantom_params=DEFAULT_PHANTOM_PARAMS, + priors=DEFAULT_PRIORS, + + iterations=5, + nwalkers=20, + burn_frac=0.10, dt=0.005, tsample=0.060, - diagnostics=None, - param_names=None, - opt_quantity=None, - priors=None, - phantoms=False, - phantom_params=None, - model_kwargs=None, - - iterations=20, - nwalkers=50, - burn_frac=0.10, - tstart=0.02, - tend=0.10, - dt=0.01, - tsample=0.06, - - sample_high_density=True, - - mds_write=False, + mds_write=True, plot=True, + phantoms=False, sample_high_density=False, + model_kwargs= { "background": 100} ) + results = run(filepath="./results/test/",) - test = run(filepath="./results/test/") - - flattest = flatdict.FlatDict(test, delimiter=".") - for key in flattest.keys(): - print(key) From d8f9be559593a4515139e091936ef47cba5f6362 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 14 Jul 2023 13:42:55 +0100 Subject: [PATCH 084/288] added background as attribute of helike model --- indica/models/helike_spectroscopy.py | 7 ++++--- indica/workflows/bayes_workflow_example.py | 9 ++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 3079ddd5..98c34813 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -48,6 +48,7 @@ def __init__( window_lim: list = [0.394, 0.401], window_vector=None, window_masks: list = [], + background = None, ): """ Read all atomic data and initialise objects @@ -87,6 +88,7 @@ def __init__( else: self.pecs = self._set_adas_pecs() + self.background = background self.window_masks = window_masks self.window_vector = window_vector if self.window_vector is not None: @@ -581,7 +583,6 @@ def __call__( calc_rho: bool = False, minimum_lines: bool = False, moment_analysis: bool = False, - background=None, **kwargs, ): """ @@ -692,8 +693,8 @@ def __call__( ) if moment_analysis: self._calculate_temperatures() - if background is not None: - self.measured_spectra = self.measured_spectra + background + if self.background is not None: + self.measured_spectra = self.measured_spectra + self.background self._build_bckc_dictionary() return self.bckc diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index da6cbb22..756c5fcc 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -213,14 +213,17 @@ def setup_models(self, diagnostics: list): model.set_los_transform(los_transform) elif diag == "xrcs": + + self.xrcs_background = self.opt_data["xrcs.spectra"].sel(wavelength=slice(0.392, 0.388)).mean( + dim="wavelength").sel(t=self.plasma.time_to_calculate) los_transform = self.data["xrcs"]["te_kw"].transform model = Helike_spectroscopy( name="xrcs", window_masks=[slice(0.394, 0.396)], window_vector=self.data[diag][ "spectra" - ].wavelength.values - , + ].wavelength.values, + background=self.xrcs_background, ) model.set_los_transform(los_transform) @@ -387,7 +390,7 @@ def __call__(self, filepath = "./results/test/", **kwargs): plot=True, phantoms=False, sample_high_density=False, - model_kwargs= { "background": 100} + model_kwargs= {} ) results = run(filepath="./results/test/",) From 1f0a0f9f8f48a8acb916c25e09eea58aacd38be6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 14 Jul 2023 15:43:37 +0100 Subject: [PATCH 085/288] fixed phantom methods --- indica/workflows/bayes_workflow_dev.py | 27 +++-- indica/workflows/bayes_workflow_example.py | 113 ++++++++++----------- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/indica/workflows/bayes_workflow_dev.py b/indica/workflows/bayes_workflow_dev.py index 5fa34615..bf7521c1 100644 --- a/indica/workflows/bayes_workflow_dev.py +++ b/indica/workflows/bayes_workflow_dev.py @@ -74,25 +74,25 @@ } OPTIMISED_PARAMS = [ - # "Ne_prof.y1", + "Ne_prof.y1", "Ne_prof.y0", - # "Ne_prof.peaking", + "Ne_prof.peaking", # "Ne_prof.wcenter", # "Ne_prof.wped", # "ar_conc", # "Nimp_prof.y1", "Nimp_prof.y0", # "Nimp_prof.wcenter", - # "Nimp_prof.wped", - # "Nimp_prof.peaking", + "Nimp_prof.wped", + "Nimp_prof.peaking", "Te_prof.y0", # "Te_prof.wped", - # "Te_prof.wcenter", - # "Te_prof.peaking", + "Te_prof.wcenter", + "Te_prof.peaking", "Ti_prof.y0", # "Ti_prof.wped", - # "Ti_prof.wcenter", - # "Ti_prof.peaking", + "Ti_prof.wcenter", + "Ti_prof.peaking", ] OPTIMISED_QUANTITY = [ @@ -371,18 +371,17 @@ def _exp_data(self): def __call__(self, filepath = "./results/test/", **kwargs): if self.mds_write: - check_analysis_run(self.pulse, self.run) + # check_analysis_run(self.pulse, self.run) self.node_structure = create_nodes(pulse_to_write=self.pulse_to_write, diagnostic_quantities=self.opt_quantity, mode="NEW") self.run_sampler() self.save_pickle(filepath=filepath) - self.result = self.dict_of_dataarray_to_numpy(self.result) - if self.plot: plot_bayes_result(self.result, filepath) + self.result = self.dict_of_dataarray_to_numpy(self.result) if self.mds_write: write_nodes(self.pulse_to_write, self.node_structure, self.result) @@ -401,14 +400,14 @@ def __call__(self, filepath = "./results/test/", **kwargs): phantom_params=DEFAULT_PHANTOM_PARAMS, priors=DEFAULT_PRIORS, - iterations=5, - nwalkers=20, + iterations=100, + nwalkers=50, burn_frac=0.10, dt=0.005, tsample=0.060, mds_write=True, - plot=False, + plot=True, phantoms=False, sample_high_density=False, model_kwargs= { "background": 100} diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 756c5fcc..e476504c 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -74,25 +74,25 @@ } OPTIMISED_PARAMS = [ - # "Ne_prof.y1", + "Ne_prof.y1", "Ne_prof.y0", - # "Ne_prof.peaking", + "Ne_prof.peaking", # "Ne_prof.wcenter", # "Ne_prof.wped", # "ar_conc", # "Nimp_prof.y1", "Nimp_prof.y0", # "Nimp_prof.wcenter", - # "Nimp_prof.wped", - # "Nimp_prof.peaking", + "Nimp_prof.wped", + "Nimp_prof.peaking", "Te_prof.y0", # "Te_prof.wped", - # "Te_prof.wcenter", - # "Te_prof.peaking", + "Te_prof.wcenter", + "Te_prof.peaking", "Ti_prof.y0", # "Ti_prof.wped", - # "Ti_prof.wcenter", - # "Ti_prof.peaking", + "Ti_prof.wcenter", + "Ti_prof.peaking", ] OPTIMISED_QUANTITY = [ @@ -102,7 +102,7 @@ "smmh1.ne"] -class DevBayesWorkflow(AbstractBayesWorkflow): +class ExampleBayesWorkflow(AbstractBayesWorkflow): def __init__( self, pulse=None, @@ -158,8 +158,8 @@ def __init__( self.setup_plasma() self.save_phantom_profiles() self.read_data(self.diagnostics) - self.setup_opt_data(phantoms=self.phantoms) self.setup_models(self.diagnostics) + self.setup_opt_data(phantoms=self.phantoms) self.setup_optimiser(self.model_kwargs) def setup_plasma(self): @@ -171,7 +171,7 @@ def setup_plasma(self): impurities=("ar", "c"), impurity_concentration=( 0.001, - 0.02, + 0.04, ), full_run=False, n_rad=20, @@ -182,12 +182,6 @@ def setup_plasma(self): self.plasma.update_profiles(self.phantom_params) self.plasma.build_atomic_data(calc_power_loss=False) - def setup_opt_data(self, phantoms=False): - if phantoms: - self._phantom_data() - else: - self._exp_data() - def setup_models(self, diagnostics: list): self.models = {} for diag in self.diagnostics: @@ -213,17 +207,13 @@ def setup_models(self, diagnostics: list): model.set_los_transform(los_transform) elif diag == "xrcs": - - self.xrcs_background = self.opt_data["xrcs.spectra"].sel(wavelength=slice(0.392, 0.388)).mean( - dim="wavelength").sel(t=self.plasma.time_to_calculate) los_transform = self.data["xrcs"]["te_kw"].transform model = Helike_spectroscopy( name="xrcs", window_masks=[slice(0.394, 0.396)], window_vector=self.data[diag][ "spectra" - ].wavelength.values, - background=self.xrcs_background, + ].wavelength.values * 0.1, ) model.set_los_transform(los_transform) @@ -242,33 +232,11 @@ def setup_models(self, diagnostics: list): self.models[diag] = model - def setup_optimiser(self, model_kwargs): - - self.bayesopt = BayesModels( - plasma=self.plasma, - data=self.opt_data, - diagnostic_models=[*self.models.values()], - quant_to_optimise=self.opt_quantity, - priors=self.priors, - ) - - ndim = len(self.param_names) - self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] - self.sampler = emcee.EnsembleSampler( - self.nwalkers, - ndim, - log_prob_fn=self.bayesopt.ln_posterior, - parameter_names=self.param_names, - moves=self.move, - kwargs=model_kwargs, - ) - - if self.sample_high_density: - self.start_points = self.bayesopt.sample_from_high_density_region(self.param_names, self.sampler, self.nwalkers) + def setup_opt_data(self, phantoms=False): + if phantoms: + self._phantom_data() else: - self.start_points = self.bayesopt.sample_from_priors( - self.param_names, size=self.nwalkers - ) + self._exp_data() def _phantom_data(self, noise=False, noise_factor=0.1): self.opt_data = {} @@ -279,19 +247,22 @@ def _phantom_data(self, noise=False, noise_factor=0.1): .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if "xrcs" in self.diagnostics: + self.opt_data["xrcs.spectra"] = ( self.models["xrcs"]() .pop("spectra") .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) - self.opt_data["xrcs.background"] = None + self.opt_data["xrcs.spectra"]["error"] = np.sqrt( + self.opt_data["xrcs.spectra"] + ) if "cxff_pi" in self.diagnostics: cxrs_data = ( self.models["cxff_pi"]() .pop("ti") .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) - self.opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) + self.opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2, drop=True) if "efit" in self.diagnostics: self.opt_data["efit.wp"] = ( self.models["efit"]() @@ -337,7 +308,7 @@ def _exp_data(self): & (self.opt_data["xrcs.spectra"].wavelength > 0.388), drop=True, ) - self.opt_data["xrcs.background"] = background.mean(dim="wavelength") + self.models["xrcs"].background = background.mean(dim="wavelength").sel(t=self.plasma.time_to_calculate) self.opt_data["xrcs.spectra"]["error"] = np.sqrt( self.opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 ) @@ -347,6 +318,34 @@ def _exp_data(self): self.opt_data["cxff_pi"]["ti"].channel == 2, drop=True ) + def setup_optimiser(self, model_kwargs): + + self.bayesopt = BayesModels( + plasma=self.plasma, + data=self.opt_data, + diagnostic_models=[*self.models.values()], + quant_to_optimise=self.opt_quantity, + priors=self.priors, + ) + + ndim = len(self.param_names) + self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] + self.sampler = emcee.EnsembleSampler( + self.nwalkers, + ndim, + log_prob_fn=self.bayesopt.ln_posterior, + parameter_names=self.param_names, + moves=self.move, + kwargs=model_kwargs, + ) + + if self.sample_high_density: + self.start_points = self.bayesopt.sample_from_high_density_region(self.param_names, self.sampler, self.nwalkers) + else: + self.start_points = self.bayesopt.sample_from_priors( + self.param_names, size=self.nwalkers + ) + def __call__(self, filepath = "./results/test/", **kwargs): if self.mds_write: @@ -358,7 +357,7 @@ def __call__(self, filepath = "./results/test/", **kwargs): self.run_sampler() self.save_pickle(filepath=filepath) - if self.plot: # currently requires result + if self.plot: # currently requires result with DataArrays plot_bayes_result(self.result, filepath) self.result = self.dict_of_dataarray_to_numpy(self.result) @@ -370,7 +369,7 @@ def __call__(self, filepath = "./results/test/", **kwargs): if __name__ == "__main__": - run = DevBayesWorkflow( + run = ExampleBayesWorkflow( pulse=10009, pulse_to_write=23000101, run="RUN01", @@ -380,16 +379,16 @@ def __call__(self, filepath = "./results/test/", **kwargs): phantom_params=DEFAULT_PHANTOM_PARAMS, priors=DEFAULT_PRIORS, - iterations=5, - nwalkers=20, + iterations=200, + nwalkers=50, burn_frac=0.10, dt=0.005, tsample=0.060, mds_write=True, plot=True, - phantoms=False, - sample_high_density=False, + phantoms=True, + sample_high_density=True, model_kwargs= {} ) results = run(filepath="./results/test/",) From 7623a5449d5d6b543fc614593addc091c676fa2c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 14 Jul 2023 15:44:02 +0100 Subject: [PATCH 086/288] rearranged methods --- indica/workflows/abstract_bayes_workflow.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 86d1fb99..b4282ccd 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -27,8 +27,8 @@ def __init__(self, self.setup_plasma() self.save_phantom_profiles() self.read_data(self.diagnostics) - self.setup_opt_data(self.phantoms) self.setup_models(self.diagnostics) + self.setup_opt_data(self.phantoms) self.setup_optimiser() @abstractmethod @@ -47,14 +47,6 @@ def read_data(self, diagnostics: list): self.plasma.set_equilibrium(self.reader.equilibrium) self.data = self.reader.binned_data - - @abstractmethod - def setup_opt_data(self, phantom: bool = False): - """ - Prepare the data in necessary format for optimiser i.e. flat dictionary - """ - self.opt_data = {} - @abstractmethod def setup_models(self, diagnostics: list): """ @@ -63,6 +55,13 @@ def setup_models(self, diagnostics: list): """ self.models = {} + @abstractmethod + def setup_opt_data(self, phantom: bool = False): + """ + Prepare the data in necessary format for optimiser i.e. flat dictionary + """ + self.opt_data = {} + @abstractmethod def setup_optimiser(self, model_kwargs): """ From b3b7a264f53fb2e8c4ed28a8dcf461e6b52dc250 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 10:24:52 +0100 Subject: [PATCH 087/288] moved background to call kwarg --- indica/models/helike_spectroscopy.py | 7 +++---- tests/unit/workflows/__init__.py | 0 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 tests/unit/workflows/__init__.py diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 98c34813..c4e1b869 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -48,7 +48,6 @@ def __init__( window_lim: list = [0.394, 0.401], window_vector=None, window_masks: list = [], - background = None, ): """ Read all atomic data and initialise objects @@ -88,7 +87,6 @@ def __init__( else: self.pecs = self._set_adas_pecs() - self.background = background self.window_masks = window_masks self.window_vector = window_vector if self.window_vector is not None: @@ -583,6 +581,7 @@ def __call__( calc_rho: bool = False, minimum_lines: bool = False, moment_analysis: bool = False, + background: int = None, **kwargs, ): """ @@ -693,8 +692,8 @@ def __call__( ) if moment_analysis: self._calculate_temperatures() - if self.background is not None: - self.measured_spectra = self.measured_spectra + self.background + if background is not None: + self.measured_spectra = self.measured_spectra + background self._build_bckc_dictionary() return self.bckc diff --git a/tests/unit/workflows/__init__.py b/tests/unit/workflows/__init__.py new file mode 100644 index 00000000..e69de29b From 4a7e022981e11dbd924d2ec94376259ca733f181 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 11:20:52 +0100 Subject: [PATCH 088/288] params and kwargs now are given as model_{var} and model prefix is removed before model call --- indica/bayesmodels.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index d6092f09..019e18e3 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -61,7 +61,7 @@ def __init__( if missing_data: # gives list of keys in quant_to_optimise but not data raise ValueError(f"{missing_data} not found in data given") - def _build_bckc(self, params: dict, **kwargs): + def _build_bckc(self, params, **kwargs): """ TODO: consider how to handle when models have overlapping kwargs Parameters @@ -73,13 +73,19 @@ def _build_bckc(self, params: dict, **kwargs): ------- bckc of results """ - # Params is a dictionary which is updated by optimiser, - # kwargs is constant i.e. settings for models self.bckc: dict = {} for model in self.diagnostic_models: - self.bckc = dict( - self.bckc, **{model.name: {**model(**{**params, **kwargs})}} - ) + # removes "model.name_" from params and kwargs then passes them to model e.g. xrcs_background -> background + _nuisance_params = {param_name.replace(model.name+"_", ""): + param_value for param_name, param_value in params.items() + if model.name in param_name} + _model_kwargs = {kwarg_name.replace(model.name+"_", ""): + kwarg_value for kwarg_name, kwarg_value in kwargs.items() + if model.name in kwarg_name} + + model_bckc = {model.name: {**model(**{**_nuisance_params, **_model_kwargs})}} + self.bckc = dict(self.bckc, **model_bckc) + self.bckc = flatdict.FlatDict(self.bckc, delimiter=".") return From e78f63814a34e8b7c58fca08bcc015e585055206 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 13:19:09 +0100 Subject: [PATCH 089/288] adding _phantom_data and _exp_data abstract methods --- indica/workflows/abstract_bayes_workflow.py | 37 ++++++++++++++------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index b4282ccd..a8754f6d 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -24,29 +24,29 @@ def __init__(self, self.opt_quantity = opt_quantity self.priors = priors + self.read_data(self.diagnostics) self.setup_plasma() self.save_phantom_profiles() - self.read_data(self.diagnostics) self.setup_models(self.diagnostics) self.setup_opt_data(self.phantoms) self.setup_optimiser() - @abstractmethod - def setup_plasma(self): - """ - Contains all methods and settings for plasma object to be used in optimisation - """ - self.plasma = None - def read_data(self, diagnostics: list): self.reader = ReadST40( self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt ) self.reader(diagnostics) - self.plasma.set_equilibrium(self.reader.equilibrium) self.data = self.reader.binned_data + @abstractmethod + def setup_plasma(self): + """ + Contains all methods and settings for setting up / initialising plasma object + """ + self.plasma = None + self.plasma.set_equilibrium(self.reader.equilibrium) + @abstractmethod def setup_models(self, diagnostics: list): """ @@ -55,12 +55,25 @@ def setup_models(self, diagnostics: list): """ self.models = {} + @abstractmethod + def _phantom_data(self): + opt_data = {} + return opt_data + + @abstractmethod + def _exp_data(self): + opt_data = {} + return opt_data + @abstractmethod def setup_opt_data(self, phantom: bool = False): """ - Prepare the data in necessary format for optimiser i.e. flat dictionary + Get and prepare the data in necessary format for optimiser """ - self.opt_data = {} + if phantom: + self.opt_data = self._phantom_data() + else: + self.opt_data = self._exp_data() @abstractmethod def setup_optimiser(self, model_kwargs): @@ -82,7 +95,7 @@ def save_phantom_profiles(self, kinetic_profiles=None): t=self.plasma.time_to_calculate) * 0 for profile_key in kinetic_profiles} - self.phantom_profiles = phantom_profiles + self.phantom_profiles = phantom_profiles def _build_result_dict(self): From 82d6fee53c41c21ea51fe8c64a44aa05f22cbaea Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 13:20:28 +0100 Subject: [PATCH 090/288] renaming to BayesWorkflowExample --- indica/workflows/bayes_workflow_example.py | 169 ++++++++++----------- 1 file changed, 81 insertions(+), 88 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index e476504c..6b598060 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -74,59 +74,60 @@ } OPTIMISED_PARAMS = [ - "Ne_prof.y1", + # "Ne_prof.y1", "Ne_prof.y0", - "Ne_prof.peaking", + # "Ne_prof.peaking", # "Ne_prof.wcenter", # "Ne_prof.wped", # "ar_conc", # "Nimp_prof.y1", "Nimp_prof.y0", # "Nimp_prof.wcenter", - "Nimp_prof.wped", - "Nimp_prof.peaking", + # "Nimp_prof.wped", + # "Nimp_prof.peaking", "Te_prof.y0", # "Te_prof.wped", "Te_prof.wcenter", "Te_prof.peaking", "Ti_prof.y0", - # "Ti_prof.wped", + "Ti_prof.wped", "Ti_prof.wcenter", "Ti_prof.peaking", ] OPTIMISED_QUANTITY = [ - "xrcs.spectra", - "cxff_pi.ti", - "efit.wp", - "smmh1.ne"] + "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne", +] -class ExampleBayesWorkflow(AbstractBayesWorkflow): +class BayesWorkflowExample(AbstractBayesWorkflow): def __init__( self, - pulse=None, - pulse_to_write=None, - run="RUN01", - diagnostics=None, - param_names=None, - opt_quantity=None, - priors = None, - phantoms=False, - phantom_params=None, - model_kwargs = None, - - nwalkers=50, - tstart=0.02, - tend=0.10, - dt=0.01, - tsample=0.06, - iterations=100, - burn_frac=0, - - mds_write=False, - plot=True, - sample_high_density = False, + pulse: int = None, + pulse_to_write: bool = None, + run: str = "RUN01", + diagnostics: list = None, + param_names: list = None, + opt_quantity: list = None, + priors: dict = None, + profile_params: dict = None, + phantoms: bool = False, + model_kwargs: dict = {}, + + nwalkers: int = 50, + tstart: float = 0.02, + tend: float = 0.10, + dt: float = 0.01, + tsample: float = 0.06, + iterations: int = 100, + burn_frac: float = 0, + + mds_write: bool = False, + plot: bool = True, + sample_high_density: bool = False, ): self.pulse = pulse self.pulse_to_write = pulse_to_write @@ -135,7 +136,7 @@ def __init__( self.param_names = param_names self.opt_quantity = opt_quantity self.priors = priors - self.phantom_params = phantom_params + self.profile_params = profile_params self.model_kwargs = model_kwargs self.phantoms = phantoms @@ -151,13 +152,13 @@ def __init__( self.plot = plot self.sample_high_density = sample_high_density - for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics", "phantom_params"]: + for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics", "profile_params"]: if getattr(self, attribute) is None: raise ValueError(f"{attribute} needs to be defined") + self.read_data(self.diagnostics) self.setup_plasma() self.save_phantom_profiles() - self.read_data(self.diagnostics) self.setup_models(self.diagnostics) self.setup_opt_data(phantoms=self.phantoms) self.setup_optimiser(self.model_kwargs) @@ -179,7 +180,8 @@ def setup_plasma(self): self.plasma.time_to_calculate = self.plasma.t[ np.abs(self.tsample - self.plasma.t).argmin() ] - self.plasma.update_profiles(self.phantom_params) + self.plasma.set_equilibrium(self.reader.equilibrium) + self.plasma.update_profiles(self.profile_params) self.plasma.build_atomic_data(calc_power_loss=False) def setup_models(self, diagnostics: list): @@ -228,33 +230,30 @@ def setup_models(self, diagnostics: list): else: raise ValueError(f"{diag} not found in setup_models") model.plasma = self.plasma - - self.models[diag] = model def setup_opt_data(self, phantoms=False): if phantoms: - self._phantom_data() + self.opt_data = self._phantom_data() else: - self._exp_data() + self.opt_data = self._exp_data() def _phantom_data(self, noise=False, noise_factor=0.1): - self.opt_data = {} + opt_data = {} if "smmh1" in self.diagnostics: - self.opt_data["smmh1.ne"] = ( + opt_data["smmh1.ne"] = ( self.models["smmh1"]() .pop("ne") .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if "xrcs" in self.diagnostics: - - self.opt_data["xrcs.spectra"] = ( + opt_data["xrcs.spectra"] = ( self.models["xrcs"]() .pop("spectra") .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) - self.opt_data["xrcs.spectra"]["error"] = np.sqrt( - self.opt_data["xrcs.spectra"] + opt_data["xrcs.spectra"]["error"] = np.sqrt( + opt_data["xrcs.spectra"] ) if "cxff_pi" in self.diagnostics: cxrs_data = ( @@ -262,61 +261,56 @@ def _phantom_data(self, noise=False, noise_factor=0.1): .pop("ti") .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) - self.opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2, drop=True) + opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2, drop=True) if "efit" in self.diagnostics: - self.opt_data["efit.wp"] = ( + opt_data["efit.wp"] = ( self.models["efit"]() .pop("wp") .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if noise: - self.opt_data["smmh1.ne"] = self.opt_data["smmh1.ne"] + self.opt_data[ + opt_data["smmh1.ne"] = opt_data["smmh1.ne"] + self.opt_data[ "smmh1.ne" ].max().values * np.random.normal(0, noise_factor, None) - self.opt_data["xrcs.spectra"] = self.opt_data[ - "xrcs.spectra" - ] + np.random.normal( + opt_data["xrcs.spectra"] = opt_data[ + "xrcs.spectra" + ] + np.random.normal( 0, - np.sqrt(self.opt_data["xrcs.spectra"].values[0,]), - self.opt_data["xrcs.spectra"].shape[1], + np.sqrt(opt_data["xrcs.spectra"].values[0,]), + opt_data["xrcs.spectra"].shape[1], ) - self.opt_data["cxff_pi.ti"] = self.opt_data[ - "cxff_pi.ti" - ] + self.opt_data["cxff_pi.ti"].max().values * np.random.normal( - 0, noise_factor, self.opt_data["cxff_pi.ti"].shape[1] + opt_data["cxff_pi.ti"] = opt_data[ + "cxff_pi.ti" + ] + opt_data["cxff_pi.ti"].max().values * np.random.normal( + 0, noise_factor, opt_data["cxff_pi.ti"].shape[1] ) - self.opt_data["efit.wp"] = self.opt_data["efit.wp"] + self.opt_data[ + opt_data["efit.wp"] = opt_data["efit.wp"] + opt_data[ "efit.wp" ].max().values * np.random.normal(0, noise_factor, None) - - self.phantom_profiles = {} - for key in ["electron_density", "impurity_density", "electron_temperature", "ion_temperature", - "ion_density", "fast_density", "neutral_density"]: - self.phantom_profiles[key] = getattr(self.plasma, key).sel( - t=self.plasma.time_to_calculate - ).copy() + return opt_data def _exp_data(self): - self.opt_data = flatdict.FlatDict(self.data, ".") + opt_data = flatdict.FlatDict(self.data, ".") if "xrcs" in self.diagnostics: - self.opt_data["xrcs.spectra"]["wavelength"] = ( - self.opt_data["xrcs.spectra"].wavelength * 0.1 + opt_data["xrcs.spectra"]["wavelength"] = ( + opt_data["xrcs.spectra"].wavelength * 0.1 ) - background = self.opt_data["xrcs.spectra"].where( - (self.opt_data["xrcs.spectra"].wavelength < 0.392) - & (self.opt_data["xrcs.spectra"].wavelength > 0.388), + background = opt_data["xrcs.spectra"].where( + (opt_data["xrcs.spectra"].wavelength < 0.392) + & (opt_data["xrcs.spectra"].wavelength > 0.388), drop=True, ) - self.models["xrcs"].background = background.mean(dim="wavelength").sel(t=self.plasma.time_to_calculate) - self.opt_data["xrcs.spectra"]["error"] = np.sqrt( - self.opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 + self.model_kwargs["xrcs_background"] = background.mean(dim="wavelength").sel(t=self.plasma.time_to_calculate) + opt_data["xrcs.spectra"]["error"] = np.sqrt( + opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 ) if "cxff_pi" in self.diagnostics: - self.opt_data["cxff_pi"]["ti"] = self.opt_data["cxff_pi"]["ti"].where( - self.opt_data["cxff_pi"]["ti"].channel == 2, drop=True + opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( + opt_data["cxff_pi"]["ti"].channel == 2, drop=True ) + return opt_data def setup_optimiser(self, model_kwargs): @@ -340,13 +334,14 @@ def setup_optimiser(self, model_kwargs): ) if self.sample_high_density: - self.start_points = self.bayesopt.sample_from_high_density_region(self.param_names, self.sampler, self.nwalkers) + self.start_points = self.bayesopt.sample_from_high_density_region(self.param_names, self.sampler, + self.nwalkers) else: self.start_points = self.bayesopt.sample_from_priors( self.param_names, size=self.nwalkers ) - def __call__(self, filepath = "./results/test/", **kwargs): + def __call__(self, filepath="./results/test/", **kwargs): if self.mds_write: # check_analysis_run(self.pulse, self.run) @@ -368,19 +363,19 @@ def __call__(self, filepath = "./results/test/", **kwargs): if __name__ == "__main__": - - run = ExampleBayesWorkflow( + run = BayesWorkflowExample( pulse=10009, pulse_to_write=23000101, run="RUN01", diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], opt_quantity=OPTIMISED_QUANTITY, param_names=OPTIMISED_PARAMS, - phantom_params=DEFAULT_PHANTOM_PARAMS, + profile_params=DEFAULT_PHANTOM_PARAMS, priors=DEFAULT_PRIORS, + model_kwargs={"xrcs_moment_analysis":False, }, - iterations=200, - nwalkers=50, + iterations=10, + nwalkers=20, burn_frac=0.10, dt=0.005, tsample=0.060, @@ -388,8 +383,6 @@ def __call__(self, filepath = "./results/test/", **kwargs): mds_write=True, plot=True, phantoms=True, - sample_high_density=True, - model_kwargs= {} + sample_high_density=False, ) - results = run(filepath="./results/test/",) - + results = run(filepath="./results/test/", ) From 2c1c48aa3157a90771d44028a8afa08b05b363f7 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 13:22:43 +0100 Subject: [PATCH 091/288] initial commit --- .../workflows/test_bayes_workflow_example.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/unit/workflows/test_bayes_workflow_example.py diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py new file mode 100644 index 00000000..f08ca42e --- /dev/null +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -0,0 +1,61 @@ +from indica.workflows.bayes_workflow_example import ExampleBayesWorkflow + + +""" +IDEAS: + +including abstract class tests in here... + +if save phantom profiles is mutable + +does it initialise + +setup models: + does it run + give wrong diagnostic name + give duplicate names + give correct names + +_phantom_data: + does it run +check dictionary: + all have time dim + all exist + + with noise + all have error? + shape + +_exp_data: + does it run + test if no data + test shape of data + all have dim t + +setup_optimiser + does it run + start points + contains data + dims of start points + does sample from high density work + +call + does it run + are results all np.array + +""" + +class TestBayesWorkflowExample: + def setup_class(self): + return + + def setup_method(self): + return + + def teardown_method(self): + return + + def test_plasma_initializes(self): + assert hasattr(self, "plasma") + + From c7a2ca58b87ec257b5a536a1415b1b220b176ba5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 13:24:22 +0100 Subject: [PATCH 092/288] renamed phantom_params to profile_params --- indica/workflows/bayes_workflow_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 6b598060..b7733de3 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -18,7 +18,7 @@ from indica.writers.bda_tree import create_nodes, write_nodes, check_analysis_run # global configurations -DEFAULT_PHANTOM_PARAMS = { +DEFAULT_PROFILE_PARAMS = { "Ne_prof.y0": 5e19, "Ne_prof.wcenter": 0.4, "Ne_prof.peaking": 2, @@ -370,7 +370,7 @@ def __call__(self, filepath="./results/test/", **kwargs): diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], opt_quantity=OPTIMISED_QUANTITY, param_names=OPTIMISED_PARAMS, - profile_params=DEFAULT_PHANTOM_PARAMS, + profile_params=DEFAULT_PROFILE_PARAMS, priors=DEFAULT_PRIORS, model_kwargs={"xrcs_moment_analysis":False, }, From db25e708890df372c71ee308321a9ac4e385119f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 13:56:05 +0100 Subject: [PATCH 093/288] renamed bayesopt -> bayesmodel --- indica/workflows/bayes_workflow_example.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index b7733de3..7739723f 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -314,7 +314,7 @@ def _exp_data(self): def setup_optimiser(self, model_kwargs): - self.bayesopt = BayesModels( + self.bayesmodel = BayesModels( plasma=self.plasma, data=self.opt_data, diagnostic_models=[*self.models.values()], @@ -327,7 +327,7 @@ def setup_optimiser(self, model_kwargs): self.sampler = emcee.EnsembleSampler( self.nwalkers, ndim, - log_prob_fn=self.bayesopt.ln_posterior, + log_prob_fn=self.bayesmodel.ln_posterior, parameter_names=self.param_names, moves=self.move, kwargs=model_kwargs, @@ -373,6 +373,7 @@ def __call__(self, filepath="./results/test/", **kwargs): profile_params=DEFAULT_PROFILE_PARAMS, priors=DEFAULT_PRIORS, model_kwargs={"xrcs_moment_analysis":False, }, + phantoms=True, iterations=10, nwalkers=20, @@ -382,7 +383,6 @@ def __call__(self, filepath="./results/test/", **kwargs): mds_write=True, plot=True, - phantoms=True, sample_high_density=False, ) results = run(filepath="./results/test/", ) From b1c5bd375a749e32dff8795d26d913326368c488 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 13:57:24 +0100 Subject: [PATCH 094/288] renamed bayesopt -> bayesmodel --- indica/workflows/bayes_workflow_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 7739723f..170a08f3 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -375,8 +375,8 @@ def __call__(self, filepath="./results/test/", **kwargs): model_kwargs={"xrcs_moment_analysis":False, }, phantoms=True, - iterations=10, - nwalkers=20, + iterations=200, + nwalkers=50, burn_frac=0.10, dt=0.005, tsample=0.060, From f78ce649941226ca21377f2ad00c3f1d6e2fa700 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 15:00:19 +0100 Subject: [PATCH 095/288] moved start point sampling to its own method _sample_start_points --- indica/workflows/bayes_workflow_example.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 170a08f3..cfe72309 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -332,8 +332,10 @@ def setup_optimiser(self, model_kwargs): moves=self.move, kwargs=model_kwargs, ) + self.start_points = self._sample_start_points(sample_high_density=self.sample_high_density) - if self.sample_high_density: + def _sample_start_points(self, sample_high_density: bool=True): + if sample_high_density: self.start_points = self.bayesopt.sample_from_high_density_region(self.param_names, self.sampler, self.nwalkers) else: From c3a418bd90cfcfeb66ae86786a348ef985fd5fa0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 15:04:35 +0100 Subject: [PATCH 096/288] fixed variable name --- indica/workflows/bayes_workflow_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index cfe72309..98b68e8a 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -336,10 +336,10 @@ def setup_optimiser(self, model_kwargs): def _sample_start_points(self, sample_high_density: bool=True): if sample_high_density: - self.start_points = self.bayesopt.sample_from_high_density_region(self.param_names, self.sampler, + self.start_points = self.bayesmodel.sample_from_high_density_region(self.param_names, self.sampler, self.nwalkers) else: - self.start_points = self.bayesopt.sample_from_priors( + self.start_points = self.bayesmodel.sample_from_priors( self.param_names, size=self.nwalkers ) From 196eb60f7c978f6cf896dda6ea1899090c137332 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 2 Aug 2023 15:09:49 +0100 Subject: [PATCH 097/288] fixed bug with start_points being overwritten --- indica/workflows/bayes_workflow_example.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 98b68e8a..9f610183 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -336,12 +336,13 @@ def setup_optimiser(self, model_kwargs): def _sample_start_points(self, sample_high_density: bool=True): if sample_high_density: - self.start_points = self.bayesmodel.sample_from_high_density_region(self.param_names, self.sampler, + start_points = self.bayesmodel.sample_from_high_density_region(self.param_names, self.sampler, self.nwalkers) else: - self.start_points = self.bayesmodel.sample_from_priors( + start_points = self.bayesmodel.sample_from_priors( self.param_names, size=self.nwalkers ) + return start_points def __call__(self, filepath="./results/test/", **kwargs): From 31542b0835d80af6f0c3c47e0c11768ddcce84e1 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 09:44:21 +0100 Subject: [PATCH 098/288] removed workflow_dev --- indica/workflows/bayes_workflow_dev.py | 416 ------------------------- 1 file changed, 416 deletions(-) delete mode 100644 indica/workflows/bayes_workflow_dev.py diff --git a/indica/workflows/bayes_workflow_dev.py b/indica/workflows/bayes_workflow_dev.py deleted file mode 100644 index bf7521c1..00000000 --- a/indica/workflows/bayes_workflow_dev.py +++ /dev/null @@ -1,416 +0,0 @@ -import emcee -import numpy as np -import flatdict -import copy -from scipy.stats import loguniform - -from indica.readers.read_st40 import ReadST40 -from indica.bayesmodels import BayesModels, get_uniform -from indica.workflows.bayes_plots import plot_bayes_result -from indica.models.interferometry import Interferometry -from indica.models.helike_spectroscopy import Helike_spectroscopy -from indica.models.charge_exchange import ChargeExchange -from indica.models.equilibrium_reconstruction import EquilibriumReconstruction -from indica.models.plasma import Plasma -from indica.converters.line_of_sight import LineOfSightTransform - -from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow -from indica.writers.bda_tree import create_nodes, write_nodes, check_analysis_run - -# global configurations -DEFAULT_PHANTOM_PARAMS = { - "Ne_prof.y0": 5e19, - "Ne_prof.wcenter": 0.4, - "Ne_prof.peaking": 2, - "Ne_prof.y1": 2e18, - "Ne_prof.yend": 1e18, - "Ne_prof.wped": 2, - "Nimp_prof.y0": 3e16, - "Nimp_prof.y1": 0.5e16, - "Nimp_prof.wcenter": 0.4, - "Nimp_prof.wped": 6, - "Nimp_prof.peaking": 2, - "Te_prof.y0": 3000, - "Te_prof.y1": 50, - "Te_prof.wcenter": 0.4, - "Te_prof.wped": 3, - "Te_prof.peaking": 2, - "Ti_prof.y0": 6000, - "Ti_prof.y1": 50, - "Ti_prof.wcenter": 0.4, - "Ti_prof.wped": 3, - "Ti_prof.peaking": 2, -} - -DEFAULT_PRIORS = { - "Ne_prof.y0": get_uniform(2e19, 4e20), - "Ne_prof.y1": get_uniform(1e18, 1e19), - "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), - "Ne_prof.wped": get_uniform(1, 6), - "Ne_prof.wcenter": get_uniform(0.1, 0.8), - "Ne_prof.peaking": get_uniform(1, 6), - "ar_conc": loguniform(0.0001, 0.01), - "Nimp_prof.y0": loguniform(1e16, 1e18), - "Nimp_prof.y1": get_uniform(1e15, 2e16), - "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( - (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 - ), - "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), - "Nimp_prof.wped": get_uniform(1, 6), - "Nimp_prof.wcenter": get_uniform(0.1, 0.8), - "Nimp_prof.peaking": get_uniform(1, 20), - "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where( - (x1 > x2), 1, 0 - ), # impurity always more peaked - "Te_prof.y0": get_uniform(1000, 5000), - "Te_prof.wped": get_uniform(1, 6), - "Te_prof.wcenter": get_uniform(0.1, 0.6), - "Te_prof.peaking": get_uniform(1, 6), - "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode - "Ti_prof.y0": get_uniform(3000, 10000), - "Ti_prof.wped": get_uniform(1, 6), - "Ti_prof.wcenter": get_uniform(0.1, 0.6), - "Ti_prof.peaking": get_uniform(1, 20), -} - -OPTIMISED_PARAMS = [ - "Ne_prof.y1", - "Ne_prof.y0", - "Ne_prof.peaking", - # "Ne_prof.wcenter", - # "Ne_prof.wped", - # "ar_conc", - # "Nimp_prof.y1", - "Nimp_prof.y0", - # "Nimp_prof.wcenter", - "Nimp_prof.wped", - "Nimp_prof.peaking", - "Te_prof.y0", - # "Te_prof.wped", - "Te_prof.wcenter", - "Te_prof.peaking", - "Ti_prof.y0", - # "Ti_prof.wped", - "Ti_prof.wcenter", - "Ti_prof.peaking", -] - -OPTIMISED_QUANTITY = [ - "xrcs.spectra", - "cxff_pi.ti", - "efit.wp", - "smmh1.ne"] - - -class DevBayesWorkflow(AbstractBayesWorkflow): - def __init__( - self, - pulse=None, - pulse_to_write=None, - run="RUN01", - diagnostics=None, - param_names=None, - opt_quantity=None, - priors = None, - phantoms=False, - phantom_params=None, - model_kwargs = None, - - nwalkers=50, - tstart=0.02, - tend=0.10, - dt=0.01, - tsample=0.06, - iterations=100, - burn_frac=0, - - mds_write=False, - plot=True, - sample_high_density = False, - fast_particles = False, - ): - self.pulse = pulse - self.pulse_to_write = pulse_to_write - self.run = run - self.diagnostics = diagnostics - self.param_names = param_names - self.opt_quantity = opt_quantity - self.priors = priors - self.phantom_params = phantom_params - self.model_kwargs = model_kwargs - self.phantoms = phantoms - - self.tstart = tstart - self.tend = tend - self.dt = dt - self.tsample = tsample - self.nwalkers = nwalkers - self.iterations = iterations - self.burn_frac = burn_frac - - self.mds_write = mds_write - self.plot = plot - self.sample_high_density = sample_high_density - self.fast_particles = fast_particles - - for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics", "phantom_params"]: - if getattr(self, attribute) is None: - raise ValueError(f"{attribute} needs to be defined") - - self.setup_plasma() - self.save_phantom_profiles() - self.read_data(self.diagnostics) - self.setup_opt_data(phantoms=self.phantoms) - self.setup_models(self.diagnostics) - self.setup_optimiser(self.model_kwargs) - - def setup_plasma(self): - self.plasma = Plasma( - tstart=self.tstart, - tend=self.tend, - dt=self.dt, - main_ion="h", - impurities=("ar", "c"), - impurity_concentration=( - 0.001, - 0.02, - ), - full_run=False, - n_rad=20, - ) - self.plasma.time_to_calculate = self.plasma.t[ - np.abs(self.tsample - self.plasma.t).argmin() - ] - self.plasma.update_profiles(self.phantom_params) - self.plasma.build_atomic_data(calc_power_loss=False) - if self.fast_particles: - self._init_fast_particles() - - def _init_fast_particles(self): - st40_code = ReadST40(13110009, self.tstart, self.tend, dt=self.dt, tree="astra") - st40_code.get_raw_data("", "astra", "RUN573") - st40_code.bin_data_in_time(["astra"], self.tstart, self.tend, self.dt) - code_data = st40_code.binned_data["astra"] - Nf = ( - code_data["nf"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - * 1.0e19 - ) - self.plasma.fast_density.values = Nf.values - Nn = ( - code_data["nn"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - * 1.0e19 - ) - self.plasma.neutral_density.values = Nn.values - Pblon = code_data["pblon"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - self.plasma.pressure_fast_parallel.values = Pblon.values - Pbper = code_data["pbper"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - self.plasma.pressure_fast_perpendicular.values = Pbper.values - - def setup_opt_data(self, phantoms=False): - if phantoms: - self._phantom_data() - else: - self._exp_data() - - def setup_models(self, diagnostics: list): - self.models = {} - for diag in self.diagnostics: - if diag == "smmh1": - # los_transform = self.data["smmh1"]["ne"].transform - machine_dims = self.plasma.machine_dimensions - origin = np.array([[-0.38063365, 0.91893092, 0.01]]) - # end = np.array([[0, 0, 0.01]]) - direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) - los_transform = LineOfSightTransform( - origin[:, 0], - origin[:, 1], - origin[:, 2], - direction[:, 0], - direction[:, 1], - direction[:, 2], - name="", - machine_dimensions=machine_dims, - passes=2, - ) - los_transform.set_equilibrium(self.plasma.equilibrium) - model = Interferometry(name=diag) - model.set_los_transform(los_transform) - - elif diag == "xrcs": - los_transform = self.data["xrcs"]["te_kw"].transform - model = Helike_spectroscopy( - name="xrcs", - window_masks=[slice(0.394, 0.396)], - window_vector=self.data[diag][ - "spectra" - ].wavelength.values - , - ) - model.set_los_transform(los_transform) - - elif diag == "efit": - model = EquilibriumReconstruction(name="efit") - - elif diag == "cxff_pi": - transform = self.data[diag]["ti"].transform - transform.set_equilibrium(self.plasma.equilibrium) - model = ChargeExchange(name=diag, element="ar") - model.set_transect_transform(transform) - else: - raise ValueError(f"{diag} not found in setup_models") - model.plasma = self.plasma - - - self.models[diag] = model - - def setup_optimiser(self, model_kwargs): - - self.bayesopt = BayesModels( - plasma=self.plasma, - data=self.opt_data, - diagnostic_models=[*self.models.values()], - quant_to_optimise=self.opt_quantity, - priors=self.priors, - ) - - ndim = len(self.param_names) - self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] - self.sampler = emcee.EnsembleSampler( - self.nwalkers, - ndim, - log_prob_fn=self.bayesopt.ln_posterior, - parameter_names=self.param_names, - moves=self.move, - kwargs=model_kwargs, - ) - - if self.sample_high_density: - self.start_points = self.bayesopt.sample_from_high_density_region(self.param_names, self.sampler, self.nwalkers) - else: - self.start_points = self.bayesopt.sample_from_priors( - self.param_names, size=self.nwalkers - ) - - def _phantom_data(self, noise=False, noise_factor=0.1): - self.opt_data = {} - if "smmh1" in self.diagnostics: - self.opt_data["smmh1.ne"] = ( - self.models["smmh1"]() - .pop("ne") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - ) - if "xrcs" in self.diagnostics: - self.opt_data["xrcs.spectra"] = ( - self.models["xrcs"]() - .pop("spectra") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - ) - self.opt_data["xrcs.background"] = None - if "cxff_pi" in self.diagnostics: - cxrs_data = ( - self.models["cxff_pi"]() - .pop("ti") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - ) - self.opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2) - if "efit" in self.diagnostics: - self.opt_data["efit.wp"] = ( - self.models["efit"]() - .pop("wp") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - ) - - if noise: - self.opt_data["smmh1.ne"] = self.opt_data["smmh1.ne"] + self.opt_data[ - "smmh1.ne" - ].max().values * np.random.normal(0, noise_factor, None) - self.opt_data["xrcs.spectra"] = self.opt_data[ - "xrcs.spectra" - ] + np.random.normal( - 0, - np.sqrt(self.opt_data["xrcs.spectra"].values[0,]), - self.opt_data["xrcs.spectra"].shape[1], - ) - self.opt_data["cxff_pi.ti"] = self.opt_data[ - "cxff_pi.ti" - ] + self.opt_data["cxff_pi.ti"].max().values * np.random.normal( - 0, noise_factor, self.opt_data["cxff_pi.ti"].shape[1] - ) - self.opt_data["efit.wp"] = self.opt_data["efit.wp"] + self.opt_data[ - "efit.wp" - ].max().values * np.random.normal(0, noise_factor, None) - - self.phantom_profiles = {} - for key in ["electron_density", "impurity_density", "electron_temperature", "ion_temperature", - "ion_density", "fast_density", "neutral_density"]: - self.phantom_profiles[key] = getattr(self.plasma, key).sel( - t=self.plasma.time_to_calculate - ).copy() - - def _exp_data(self): - self.opt_data = flatdict.FlatDict(self.data, ".") - if "xrcs" in self.diagnostics: - self.opt_data["xrcs.spectra"]["wavelength"] = ( - self.opt_data["xrcs.spectra"].wavelength * 0.1 - ) - background = self.opt_data["xrcs.spectra"].where( - (self.opt_data["xrcs.spectra"].wavelength < 0.392) - & (self.opt_data["xrcs.spectra"].wavelength > 0.388), - drop=True, - ) - self.opt_data["xrcs.background"] = background.mean(dim="wavelength") - self.opt_data["xrcs.spectra"]["error"] = np.sqrt( - self.opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 - ) - - if "cxff_pi" in self.diagnostics: - self.opt_data["cxff_pi"]["ti"] = self.opt_data["cxff_pi"]["ti"].where( - self.opt_data["cxff_pi"]["ti"].channel == 2, drop=True - ) - - def __call__(self, filepath = "./results/test/", **kwargs): - - if self.mds_write: - # check_analysis_run(self.pulse, self.run) - self.node_structure = create_nodes(pulse_to_write=self.pulse_to_write, - diagnostic_quantities=self.opt_quantity, - mode="NEW") - - self.run_sampler() - self.save_pickle(filepath=filepath) - if self.plot: - plot_bayes_result(self.result, filepath) - - self.result = self.dict_of_dataarray_to_numpy(self.result) - if self.mds_write: - write_nodes(self.pulse_to_write, self.node_structure, self.result) - - return self.result - - -if __name__ == "__main__": - - run = DevBayesWorkflow( - pulse=10009, - pulse_to_write=23000101, - run="RUN01", - diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], - opt_quantity=OPTIMISED_QUANTITY, - param_names=OPTIMISED_PARAMS, - phantom_params=DEFAULT_PHANTOM_PARAMS, - priors=DEFAULT_PRIORS, - - iterations=100, - nwalkers=50, - burn_frac=0.10, - dt=0.005, - tsample=0.060, - - mds_write=True, - plot=True, - phantoms=False, - sample_high_density=False, - model_kwargs= { "background": 100} - ) - results = run(filepath="./results/test/",) - From cc218b407daa1da2807cdbc06353d15685f90a00 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 09:51:28 +0100 Subject: [PATCH 099/288] moving equilibrium to workflow object --- indica/workflows/abstract_bayes_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index a8754f6d..fcbc763e 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -37,6 +37,7 @@ def read_data(self, diagnostics: list): self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt ) self.reader(diagnostics) + self.equilibrium = self.reader.equilibrium self.data = self.reader.binned_data @abstractmethod From 9cb0266276da6d75f784dbbe18e2bf73bdd6ef86 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 10:05:32 +0100 Subject: [PATCH 100/288] when reading raw data save transforms to their own attribute --- indica/readers/read_st40.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/indica/readers/read_st40.py b/indica/readers/read_st40.py index 61549965..07e95aec 100644 --- a/indica/readers/read_st40.py +++ b/indica/readers/read_st40.py @@ -75,6 +75,7 @@ def __init__( self.equilibrium: Equilibrium self.raw_data: dict = {} self.binned_data: dict = {} + self.transforms: dict = {} def reset_data(self): self.raw_data = {} @@ -104,6 +105,7 @@ def get_raw_data(self, uid: str, instrument: str, revision: RevisionLike = 0): transform = data[quant].transform if hasattr(transform, "set_equilibrium"): transform.set_equilibrium(self.equilibrium) + self.transforms[instrument] = transform self.raw_data[instrument] = data return data From 68c18722c354afe478c5bd0a4e70360257969e9b Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 10:09:14 +0100 Subject: [PATCH 101/288] transforms saved to workflow class --- indica/workflows/abstract_bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index fcbc763e..25053151 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -31,13 +31,13 @@ def __init__(self, self.setup_opt_data(self.phantoms) self.setup_optimiser() - def read_data(self, diagnostics: list): self.reader = ReadST40( self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt ) self.reader(diagnostics) self.equilibrium = self.reader.equilibrium + self.transforms = self.reader.transforms self.data = self.reader.binned_data @abstractmethod From f0f561fa8271a66caf5eee602731529bfee5843e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 10:19:04 +0100 Subject: [PATCH 102/288] fixed name --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index c4e1b869..6e334c72 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -28,7 +28,7 @@ } -class Helike_spectroscopy(DiagnosticModel): +class HelikeSpectrometer(DiagnosticModel): """ Data and methods to model XRCS spectrometer measurements From 3b2eaeee6f21aa5835adabc055daf68a17a2f3e2 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 11:30:30 +0100 Subject: [PATCH 103/288] renamed example transform --- indica/models/helike_spectroscopy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 6e334c72..d0a32d82 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -752,7 +752,7 @@ def select_transition(adf15_data, transition: str, wavelength: float): return pec -def helike_LOS_example(nchannels=3): +def helike_transform_example(nchannels=3): los_end = np.full((nchannels, 3), 0.0) los_end[:, 0] = 0.17 los_end[:, 1] = 0.0 @@ -783,9 +783,9 @@ def example_run(pulse: int = 9229, plasma=None, plot=False, **kwargs): ) # Create new diagnostic diagnostic_name = "xrcs" - los_transform = helike_LOS_example(3) + los_transform = helike_transform_example(3) los_transform.set_equilibrium(plasma.equilibrium) - model = Helike_spectroscopy( + model = HelikeSpectrometer( diagnostic_name, window_masks=[], ) From ae200d36fb69d7cfe28e6c0316d059a5d6a7e8b7 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 11:34:42 +0100 Subject: [PATCH 104/288] made example los function --- indica/models/interferometry.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/indica/models/interferometry.py b/indica/models/interferometry.py index f7c7458f..29a6b42b 100644 --- a/indica/models/interferometry.py +++ b/indica/models/interferometry.py @@ -88,20 +88,11 @@ def __call__( return self.bckc - -def example_run(pulse: int = None, plasma=None, plot=False): - if plasma is None: - plasma = example_plasma(pulse=pulse) - - # Create new interferometers diagnostics - diagnostic_name = "smmh1" +def smmh1_transform_example(): los_start = np.array([[0.8, 0, 0], [0.8, 0, -0.1], [0.8, 0, -0.2]]) los_end = np.array([[0.17, 0, 0], [0.17, 0, -0.25], [0.17, 0, -0.2]]) origin = los_start direction = los_end - los_start - model = Interferometry( - diagnostic_name, - ) los_transform = LineOfSightTransform( origin[:, 0], origin[:, 1], @@ -109,10 +100,22 @@ def example_run(pulse: int = None, plasma=None, plot=False): direction[:, 0], direction[:, 1], direction[:, 2], - name=diagnostic_name, - machine_dimensions=plasma.machine_dimensions, + name="smmh1", + machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), passes=2, ) + return los_transform + +def example_run(pulse: int = None, plasma=None, plot=False): + if plasma is None: + plasma = example_plasma(pulse=pulse) + + # Create new interferometers diagnostics + diagnostic_name = "smmh1" + model = Interferometry( + diagnostic_name, + ) + los_transform = smmh1_transform_example() los_transform.set_equilibrium(plasma.equilibrium) model.set_los_transform(los_transform) model.set_plasma(plasma) From 3d9d54078bd78bc5404532d892c4f7a6d52ea4b0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 11:35:19 +0100 Subject: [PATCH 105/288] minor name fix --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index d0a32d82..d4538b20 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -768,7 +768,7 @@ def helike_transform_example(nchannels=3): direction[0:nchannels, 0], direction[0:nchannels, 1], direction[0:nchannels, 2], - name="diagnostic_name", + name="xrcs", machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), passes=1, ) From cd04fb1c82a447485c22af617f1d2d758594af31 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 11:35:54 +0100 Subject: [PATCH 106/288] made read_test_data function --- indica/workflows/abstract_bayes_workflow.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 25053151..180a3443 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -6,7 +6,7 @@ import pickle from indica.readers.read_st40 import ReadST40 - +from indica.equilibrium import fake_equilibrium class AbstractBayesWorkflow(ABC): @abstractmethod @@ -31,6 +31,12 @@ def __init__(self, self.setup_opt_data(self.phantoms) self.setup_optimiser() + def read_test_data(self, diagnostic_transforms:dict): + # Used with phantom data for purposes of tests + self.equilibrium = fake_equilibrium(self.tstart, self.tend, self.dt, ) + self.transforms = diagnostic_transforms + self.data = {} + def read_data(self, diagnostics: list): self.reader = ReadST40( self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt From dfdd79107ad23f2315a780c1d1fb2f44c7906f0d Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 11:39:53 +0100 Subject: [PATCH 107/288] made example los function --- indica/models/charge_exchange.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/indica/models/charge_exchange.py b/indica/models/charge_exchange.py index a7cfadaf..1aae94ac 100644 --- a/indica/models/charge_exchange.py +++ b/indica/models/charge_exchange.py @@ -106,6 +106,19 @@ def __call__( return self.bckc +def pi_transform_example(nchannels:int): + x_positions = np.linspace(0.2, 0.8, nchannels) + y_positions = np.linspace(0.0, 0.0, nchannels) + z_positions = np.linspace(0.0, 0.0, nchannels) + + transect_transform = TransectCoordinates( + x_positions, + y_positions, + z_positions, + "pi", + machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), + ) + return transect_transform def example_run( pulse: int = None, @@ -120,18 +133,7 @@ def example_run( plasma = example_plasma(pulse=pulse) # Create new interferometers diagnostics - nchannels = 5 - x_positions = np.linspace(0.2, 0.8, nchannels) - y_positions = np.linspace(0.0, 0.0, nchannels) - z_positions = np.linspace(0.0, 0.0, nchannels) - - transect_transform = TransectCoordinates( - x_positions, - y_positions, - z_positions, - diagnostic_name, - machine_dimensions=plasma.machine_dimensions, - ) + transect_transform = pi_transform_example(5) transect_transform.set_equilibrium(plasma.equilibrium) model = ChargeExchange( diagnostic_name, From 4c6c1c2ab783d3832e26289948500a2c5a1437ae Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 11:51:09 +0100 Subject: [PATCH 108/288] minor type --- indica/workflows/abstract_bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 180a3443..33f4ec2f 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -299,7 +299,7 @@ def run_sampler(self): for blob_name in blob_names } self.accept_frac = self.sampler.acceptance_fraction.sum() - self.prior_sample = self.bayesopt.sample_from_priors( + self.prior_sample = self.bayesmodel.sample_from_priors( self.param_names, size=int(1e4) ) self.post_sample = self.sampler.get_chain(flat=True) From 0ce85fabe13c4446d704df111a105dff0a3b4480 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 11:54:06 +0100 Subject: [PATCH 109/288] added reader to read_test_data method --- indica/workflows/abstract_bayes_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 33f4ec2f..215742ff 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -33,6 +33,7 @@ def __init__(self, def read_test_data(self, diagnostic_transforms:dict): # Used with phantom data for purposes of tests + self.reader = None self.equilibrium = fake_equilibrium(self.tstart, self.tend, self.dt, ) self.transforms = diagnostic_transforms self.data = {} From fa5ac444ba42ef9d9984ad0dd0354bb8e905771e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 14:48:29 +0100 Subject: [PATCH 110/288] fixed copying workflow object states --- .../workflows/test_bayes_workflow_example.py | 151 ++++++++++++------ 1 file changed, 105 insertions(+), 46 deletions(-) diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index f08ca42e..a81c5cfe 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -1,61 +1,120 @@ -from indica.workflows.bayes_workflow_example import ExampleBayesWorkflow +import pytest +import copy +import numpy as np +from indica.workflows.bayes_workflow_example import BayesWorkflowExample, DEFAULT_PRIORS, DEFAULT_PROFILE_PARAMS, \ + OPTIMISED_PARAMS, OPTIMISED_QUANTITY """ -IDEAS: - -including abstract class tests in here... - -if save phantom profiles is mutable - -does it initialise - -setup models: - does it run - give wrong diagnostic name - give duplicate names - give correct names - -_phantom_data: - does it run -check dictionary: - all have time dim - all exist - - with noise - all have error? - shape - -_exp_data: - does it run - test if no data - test shape of data - all have dim t - -setup_optimiser - does it run - start points - contains data - dims of start points - does sample from high density work - -call - does it run - are results all np.array +TODO: +test kwarg handling + +Mock reader for testing experimental data methods """ + class TestBayesWorkflowExample: def setup_class(self): - return + + self.default_settings = dict(pulse=None, + pulse_to_write=23000111, + run="RUN01", + diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], + opt_quantity=OPTIMISED_QUANTITY, + param_names=OPTIMISED_PARAMS, + profile_params=DEFAULT_PROFILE_PARAMS, + priors=DEFAULT_PRIORS, + model_kwargs={"xrcs_moment_analysis": False, }, + phantoms=True, + + iterations=1, + nwalkers=20, + burn_frac=0.10, + dt=0.005, + tsample=0.060, + + mds_write=False, + plot=False, + sample_high_density=False, ) + + self.untouched_workflow = BayesWorkflowExample(**self.default_settings) + self.workflow = copy.deepcopy(self.untouched_workflow) def setup_method(self): - return + self.workflow = copy.deepcopy(self.untouched_workflow) def teardown_method(self): - return + self.workflow = None + + def test_workflow_initializes(self): + attributes_to_check = ["plasma", "phantom_profiles", "data", "reader", "opt_data", "models", "bayesmodel",] + for attribute in attributes_to_check: + if not hasattr(self.workflow, attribute): + raise ValueError(f"missing {attribute} in workflow object") + assert True + + def test_init_phantoms_false_with_example_plasma(self): + with pytest.raises(ValueError): + example = BayesWorkflowExample(dict(self.default_settings, **{"phantoms": False})) + + def test_init_not_including_all_required_inputs(self): + with pytest.raises(ValueError): + example = BayesWorkflowExample(dict(self.default_settings, **{"param_names": None})) + + # def test_reader_has_read_all_diagnostic_data(self): + # assert all(diag_name in self.workflow.reader.keys() for diag_name in self.workflow.diagnostics) + + def test_plasma_has_equilibrium(self): + assert hasattr(self.workflow.plasma, "equilibrium") + + def test_phantom_profiles_are_not_mutatable(self): + phantoms = copy.deepcopy(self.workflow.phantom_profiles) + self.workflow.plasma.electron_temperature += 1 + assert (phantoms is not self.workflow.phantom_profiles) + + def test_setup_models_with_wrong_diagnostic_names(self): + with pytest.raises(Exception): + self.workflow.setup_models(["foo", "bar", "xrcs"]) + + def test_phantom_data_exists(self): + self.workflow._phantom_data() + assert self.workflow.opt_data + + # def test_experimental_data_exists(self): + # self.workflow._exp_data() + # assert self.workflow.opt_data + + def test_phantom_data_has_time_dim(self): + self.workflow._phantom_data() + for key, value in self.workflow.opt_data.items(): + assert "t" in value.dims + + # def test_experimental_data_has_time_dim(self): + # self.workflow._exp_data() + # for key, value in self.workflow.opt_data.items(): + # assert "t" in value.dims + + def test_phantom_data_runs_with_noise_added(self): + self.workflow._phantom_data(noise=True) + assert self.workflow.opt_data + + def test_sampling_from_high_density(self): + self.workflow._sample_start_points(sample_high_density=True) + assert True + + def test_sampling_from_priors(self): + self.workflow._sample_start_points(sample_high_density=False) + assert True - def test_plasma_initializes(self): - assert hasattr(self, "plasma") + def test_worklow_has_results_after_run(self): + self.workflow.run_sampler() + if not hasattr(self.workflow, "result"): + raise ValueError(f"missing result in workflow object") + assert True +if __name__ == "__main__": + test = TestBayesWorkflowExample() + test.setup_class() + print() From 7d343a4cec6a41455fc11c1e94aa4b44004424c1 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 14:57:00 +0100 Subject: [PATCH 111/288] updated example to work with assign_profiles --- indica/models/plasma.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index fcf79388..4ce7bac6 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -1257,6 +1257,7 @@ def example_run( impurity_concentration=impurity_concentration, full_run=full_run, verbose=verbose, + n_rad=n_rad, **kwargs, ) plasma.build_atomic_data(default=True, calc_power_loss=calc_power_loss) @@ -1272,27 +1273,15 @@ def example_run( nimp_wcenter = np.linspace(0.4, 0.1, nt) for i, t in enumerate(plasma.t): plasma.Te_prof.peaking = te_peaking[i] - plasma.assign_profiles(profile="electron_temperature", t=t) - plasma.Ti_prof.peaking = te_peaking[i] plasma.Ti_prof.y0 = ti0[i] - plasma.assign_profiles(profile="ion_temperature", t=t) - plasma.Vrot_prof.peaking = vrot_peaking[i] plasma.Vrot_prof.y0 = vrot0[i] - plasma.assign_profiles(profile="toroidal_rotation", t=t) - plasma.Ne_prof.peaking = ne_peaking[i] - plasma.assign_profiles(profile="electron_density", t=t) - plasma.Nimp_prof.peaking = nimp_peaking[i] plasma.Nimp_prof.y0 = nimp_y0[i] plasma.Nimp_prof.wcenter = nimp_wcenter[i] - for elem in plasma.impurities: - plasma.assign_profiles(profile="impurity_density", t=t, element=elem) - - for elem in plasma.elements: - plasma.assign_profiles(profile="toroidal_rotation", t=t, element=elem) + plasma.assign_profiles(impurity_to_profile="ar",) if pulse is None: equilibrium_data = fake_equilibrium_data( From 489d14a4735a465fb51da9728b1e9995cfa664c5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 14:57:49 +0100 Subject: [PATCH 112/288] can now run with pulse = None and synthetic transforms/equilibrium --- indica/workflows/bayes_workflow_example.py | 91 ++++++++++++---------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 9f610183..b2d5f4e3 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -1,22 +1,19 @@ import emcee import numpy as np import flatdict -import copy from scipy.stats import loguniform -from indica.readers.read_st40 import ReadST40 from indica.bayesmodels import BayesModels, get_uniform from indica.workflows.bayes_plots import plot_bayes_result -from indica.models.interferometry import Interferometry -from indica.models.helike_spectroscopy import Helike_spectroscopy -from indica.models.charge_exchange import ChargeExchange -from indica.models.equilibrium_reconstruction import EquilibriumReconstruction -from indica.models.plasma import Plasma -from indica.converters.line_of_sight import LineOfSightTransform - from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow from indica.writers.bda_tree import create_nodes, write_nodes, check_analysis_run +from indica.models.interferometry import Interferometry, smmh1_transform_example +from indica.models.helike_spectroscopy import HelikeSpectrometer, helike_transform_example +from indica.models.charge_exchange import ChargeExchange, pi_transform_example +from indica.models.equilibrium_reconstruction import EquilibriumReconstruction +from indica.models.plasma import Plasma + # global configurations DEFAULT_PROFILE_PARAMS = { "Ne_prof.y0": 5e19, @@ -152,11 +149,20 @@ def __init__( self.plot = plot self.sample_high_density = sample_high_density - for attribute in ["pulse", "param_names", "opt_quantity", "priors", "diagnostics", "profile_params"]: + for attribute in ["param_names", "opt_quantity", "priors", "diagnostics", "profile_params"]: if getattr(self, attribute) is None: raise ValueError(f"{attribute} needs to be defined") - self.read_data(self.diagnostics) + if self.pulse is None and self.phantoms is False: + raise ValueError("Set phantoms to True when running phantom plasma i.e. pulse=None") + + if pulse is None: + print("Running in test mode") + self.read_test_data({"xrcs": helike_transform_example(1), + "smmh1": smmh1_transform_example(), + "cxff_pi": pi_transform_example(5)}) + else: + self.read_data(self.diagnostics) self.setup_plasma() self.save_phantom_profiles() self.setup_models(self.diagnostics) @@ -180,42 +186,46 @@ def setup_plasma(self): self.plasma.time_to_calculate = self.plasma.t[ np.abs(self.tsample - self.plasma.t).argmin() ] - self.plasma.set_equilibrium(self.reader.equilibrium) + self.plasma.set_equilibrium(self.equilibrium) self.plasma.update_profiles(self.profile_params) self.plasma.build_atomic_data(calc_power_loss=False) def setup_models(self, diagnostics: list): self.models = {} - for diag in self.diagnostics: + for diag in diagnostics: if diag == "smmh1": - # los_transform = self.data["smmh1"]["ne"].transform - machine_dims = self.plasma.machine_dimensions - origin = np.array([[-0.38063365, 0.91893092, 0.01]]) - # end = np.array([[0, 0, 0.01]]) - direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) - los_transform = LineOfSightTransform( - origin[:, 0], - origin[:, 1], - origin[:, 2], - direction[:, 0], - direction[:, 1], - direction[:, 2], - name="", - machine_dimensions=machine_dims, - passes=2, - ) + los_transform = self.transforms[diag] + # machine_dims = self.plasma.machine_dimensions + # origin = np.array([[-0.38063365, 0.91893092, 0.01]]) + # # end = np.array([[0, 0, 0.01]]) + # direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) + # los_transform = LineOfSightTransform( + # origin[:, 0], + # origin[:, 1], + # origin[:, 2], + # direction[:, 0], + # direction[:, 1], + # direction[:, 2], + # name="", + # machine_dimensions=machine_dims, + # passes=2, + # ) los_transform.set_equilibrium(self.plasma.equilibrium) model = Interferometry(name=diag) model.set_los_transform(los_transform) elif diag == "xrcs": - los_transform = self.data["xrcs"]["te_kw"].transform - model = Helike_spectroscopy( + los_transform = self.transforms[diag] + los_transform.set_equilibrium(self.plasma.equilibrium) + window_vector = None + if hasattr(self, "data"): + if diag in self.data.keys(): + window_vector = self.data[diag]["spectra"].wavelength.values * 0.1 + + model = HelikeSpectrometer( name="xrcs", window_masks=[slice(0.394, 0.396)], - window_vector=self.data[diag][ - "spectra" - ].wavelength.values * 0.1, + window_vector=window_vector ) model.set_los_transform(los_transform) @@ -223,7 +233,7 @@ def setup_models(self, diagnostics: list): model = EquilibriumReconstruction(name="efit") elif diag == "cxff_pi": - transform = self.data[diag]["ti"].transform + transform = self.transforms[diag] transform.set_equilibrium(self.plasma.equilibrium) model = ChargeExchange(name=diag, element="ar") model.set_transect_transform(transform) @@ -301,7 +311,8 @@ def _exp_data(self): & (opt_data["xrcs.spectra"].wavelength > 0.388), drop=True, ) - self.model_kwargs["xrcs_background"] = background.mean(dim="wavelength").sel(t=self.plasma.time_to_calculate) + self.model_kwargs["xrcs_background"] = background.mean(dim="wavelength").sel( + t=self.plasma.time_to_calculate) opt_data["xrcs.spectra"]["error"] = np.sqrt( opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 ) @@ -334,10 +345,10 @@ def setup_optimiser(self, model_kwargs): ) self.start_points = self._sample_start_points(sample_high_density=self.sample_high_density) - def _sample_start_points(self, sample_high_density: bool=True): + def _sample_start_points(self, sample_high_density: bool = True): if sample_high_density: start_points = self.bayesmodel.sample_from_high_density_region(self.param_names, self.sampler, - self.nwalkers) + self.nwalkers) else: start_points = self.bayesmodel.sample_from_priors( self.param_names, size=self.nwalkers @@ -367,7 +378,7 @@ def __call__(self, filepath="./results/test/", **kwargs): if __name__ == "__main__": run = BayesWorkflowExample( - pulse=10009, + pulse=None, pulse_to_write=23000101, run="RUN01", diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], @@ -375,7 +386,7 @@ def __call__(self, filepath="./results/test/", **kwargs): param_names=OPTIMISED_PARAMS, profile_params=DEFAULT_PROFILE_PARAMS, priors=DEFAULT_PRIORS, - model_kwargs={"xrcs_moment_analysis":False, }, + model_kwargs={"xrcs_moment_analysis": False, }, phantoms=True, iterations=200, From e04fe9beaf8dc0c832b51d59ccfab4c49f7f1165 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 14:58:19 +0100 Subject: [PATCH 113/288] formatting --- tests/unit/workflows/test_bayes_workflow_example.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index a81c5cfe..e4033134 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -117,4 +117,3 @@ def test_worklow_has_results_after_run(self): test = TestBayesWorkflowExample() test.setup_class() - print() From 6fcff7534968ff20ead6f525ff48ce0eaac88cbd Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 14:59:28 +0100 Subject: [PATCH 114/288] refactoring names --- tests/integration/models/test_helike_spectroscopy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/models/test_helike_spectroscopy.py b/tests/integration/models/test_helike_spectroscopy.py index 58440694..770275d6 100644 --- a/tests/integration/models/test_helike_spectroscopy.py +++ b/tests/integration/models/test_helike_spectroscopy.py @@ -1,5 +1,5 @@ import indica.models.helike_spectroscopy as helike -from indica.models.helike_spectroscopy import helike_LOS_example +from indica.models.helike_spectroscopy import helike_transform_example from indica.models.plasma import example_run as example_plasma @@ -8,13 +8,13 @@ def setup_class(self): self.plasma = example_plasma() # using Phantom self.single_time_point = self.plasma.time_to_calculate[1] self.multiple_time_point = self.plasma.time_to_calculate - self.multiple_channel_los_transform = helike_LOS_example(nchannels=3) - self.single_channel_los_transform = helike_LOS_example(nchannels=1) + self.multiple_channel_los_transform = helike_transform_example(nchannels=3) + self.single_channel_los_transform = helike_transform_example(nchannels=1) self.single_channel_los_transform.set_equilibrium(self.plasma.equilibrium) self.multiple_channel_los_transform.set_equilibrium(self.plasma.equilibrium) def setup_method(self): - self.model = helike.Helike_spectroscopy("diagnostic_name") + self.model = helike.HelikeSpectrometer("diagnostic_name") self.model.set_plasma(self.plasma) def test_helike_moment_runs_with_multiple_LOS( From eba8ca71a0172ef2139abada0c6a5648f30e0e4c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 15:01:02 +0100 Subject: [PATCH 115/288] deleted old bayes_models tests --- tests/integration/test_bayesmodels.py | 70 --------------------------- 1 file changed, 70 deletions(-) delete mode 100644 tests/integration/test_bayesmodels.py diff --git a/tests/integration/test_bayesmodels.py b/tests/integration/test_bayesmodels.py deleted file mode 100644 index b01f1aca..00000000 --- a/tests/integration/test_bayesmodels.py +++ /dev/null @@ -1,70 +0,0 @@ -import emcee -import flatdict - -from indica.bayesmodels import BayesModels -from indica.bayesmodels import get_uniform -from indica.models.helike_spectroscopy import helike_LOS_example -from indica.models.helike_spectroscopy import Helike_spectroscopy -from indica.models.plasma import example_run - - -class TestBayesModels: - def setup_class(self): - self.plasma = example_run() - self.plasma.time_to_calculate = self.plasma.t[1] - self.los_transform = helike_LOS_example(nchannels=1) - self.los_transform.set_equilibrium(self.plasma.equilibrium) - - def test_simple_run_bayesmodels_with_xrcs(self): - xrcs = Helike_spectroscopy( - name="xrcs", - ) - xrcs.plasma = self.plasma - xrcs.set_los_transform(self.los_transform) - - priors = { - "Te_prof.y0": get_uniform(2e3, 5e3), - "Ti_prof.y0": get_uniform(2e3, 8e3), - } - - bckc = {} - bckc = dict(bckc, **{xrcs.name: {**xrcs(calc_spectra=True)}}) - flat_phantom_data = flatdict.FlatDict(bckc, delimiter=".") - flat_phantom_data["xrcs.spectra"] = flat_phantom_data[ - "xrcs.spectra" - ].expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - - bm = BayesModels( - plasma=self.plasma, - data=flat_phantom_data, - diagnostic_models=[xrcs], - quant_to_optimise=["xrcs.spectra"], - priors=priors, - ) - - # Setup Optimiser - param_names = [ - "Te_prof.y0", - "Ti_prof.y0", - ] - - ndim = param_names.__len__() - nwalkers = ndim * 2 - start_points = bm.sample_from_priors(param_names, size=nwalkers) - - move = [emcee.moves.StretchMove()] - sampler = emcee.EnsembleSampler( - nwalkers, - ndim, - log_prob_fn=bm.ln_posterior, - parameter_names=param_names, - moves=move, - kwargs={}, - ) - sampler.run_mcmc(start_points, 10, progress=False) - - -if __name__ == "__main__": - test = TestBayesModels() - test.setup_class() - test.test_simple_run_bayesmodels_with_xrcs() From b6ece959cd5d0cfd6c167fdea3b5039a2bdbb100 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 15:20:09 +0100 Subject: [PATCH 116/288] reformatted _build_bckc for readability --- indica/bayesmodels.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 019e18e3..9be07adf 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -1,7 +1,5 @@ from copy import deepcopy import warnings - -import flatdict import numpy as np from scipy.stats import uniform @@ -79,14 +77,14 @@ def _build_bckc(self, params, **kwargs): _nuisance_params = {param_name.replace(model.name+"_", ""): param_value for param_name, param_value in params.items() if model.name in param_name} - _model_kwargs = {kwarg_name.replace(model.name+"_", ""): + _model_settings = {kwarg_name.replace(model.name+"_", ""): kwarg_value for kwarg_name, kwarg_value in kwargs.items() if model.name in kwarg_name} - model_bckc = {model.name: {**model(**{**_nuisance_params, **_model_kwargs})}} - self.bckc = dict(self.bckc, **model_bckc) - - self.bckc = flatdict.FlatDict(self.bckc, delimiter=".") + _model_kwargs = {**_nuisance_params, **_model_settings} # combine dictionaries + _bckc = model(**_model_kwargs) + _model_bckc = {f"{model.name}.{value_name}": value for value_name, value in _bckc.items()} # prepend model name to bckc + self.bckc = dict(self.bckc, **_model_bckc) return def _ln_likelihood(self): From 2bd93bfc8e110290217f03f096118a98cf930b99 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 15:22:23 +0100 Subject: [PATCH 117/288] moved percentage error to class attribute --- indica/bayesmodels.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 9be07adf..43c59268 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -45,12 +45,14 @@ def __init__( quant_to_optimise: list = [], priors: dict = {}, diagnostic_models: list = [], + percent_error: float = None, ): self.plasma = plasma self.data = data self.quant_to_optimise = quant_to_optimise self.diagnostic_models = diagnostic_models self.priors = priors + self.percent_error = percent_error for diag_model in self.diagnostic_models: diag_model.plasma = self.plasma @@ -97,7 +99,7 @@ def _ln_likelihood(self): self.data[key].sel(t=self.plasma.time_to_calculate).astype("float128") ) - exp_error = exp_data * 0.10 # Assume percentage error if none given. + exp_error = exp_data * self.percent_error # Assume percentage error if none given. if hasattr(self.data[key], "error"): if ( self.data[key].error != 0 From e46ac8ad2c1d1d4550525933ba44cd61c6efcfec Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 15:42:22 +0100 Subject: [PATCH 118/288] fixing init percent error --- indica/bayesmodels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 43c59268..cdf6bfec 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -45,7 +45,7 @@ def __init__( quant_to_optimise: list = [], priors: dict = {}, diagnostic_models: list = [], - percent_error: float = None, + percent_error: float = 0.10, ): self.plasma = plasma self.data = data From cc0aafd4ebdb9ca0191f0144634acaf82e10f28f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 8 Aug 2023 15:52:10 +0100 Subject: [PATCH 119/288] black formatting --- indica/bayesmodels.py | 69 ++-- indica/models/charge_exchange.py | 6 +- indica/models/helike_spectroscopy.py | 1 - indica/models/interferometry.py | 3 +- indica/models/plasma.py | 8 +- indica/readers/read_st40.py | 2 - indica/workflows/abstract_bayes_workflow.py | 326 ++++++++++++------ indica/workflows/bayes_plots.py | 16 +- indica/workflows/bayes_workflow_example.py | 143 ++++---- indica/writers/bda_tree.py | 100 +++--- .../workflows/test_bayes_workflow_example.py | 74 ++-- 11 files changed, 463 insertions(+), 285 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index cdf6bfec..c8801664 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -7,8 +7,15 @@ warnings.simplefilter("ignore", category=FutureWarning) -PROFILES = ["electron_temperature", "electron_density", "ion_temperature", "ion_density", - "impurity_density", "fast_density", "neutral_density"] +PROFILES = [ + "electron_temperature", + "electron_density", + "ion_temperature", + "ion_density", + "impurity_density", + "fast_density", + "neutral_density", +] def gaussian(x, mean, sigma): @@ -76,16 +83,26 @@ def _build_bckc(self, params, **kwargs): self.bckc: dict = {} for model in self.diagnostic_models: # removes "model.name_" from params and kwargs then passes them to model e.g. xrcs_background -> background - _nuisance_params = {param_name.replace(model.name+"_", ""): - param_value for param_name, param_value in params.items() - if model.name in param_name} - _model_settings = {kwarg_name.replace(model.name+"_", ""): - kwarg_value for kwarg_name, kwarg_value in kwargs.items() - if model.name in kwarg_name} - - _model_kwargs = {**_nuisance_params, **_model_settings} # combine dictionaries + _nuisance_params = { + param_name.replace(model.name + "_", ""): param_value + for param_name, param_value in params.items() + if model.name in param_name + } + _model_settings = { + kwarg_name.replace(model.name + "_", ""): kwarg_value + for kwarg_name, kwarg_value in kwargs.items() + if model.name in kwarg_name + } + + _model_kwargs = { + **_nuisance_params, + **_model_settings, + } # combine dictionaries _bckc = model(**_model_kwargs) - _model_bckc = {f"{model.name}.{value_name}": value for value_name, value in _bckc.items()} # prepend model name to bckc + _model_bckc = { + f"{model.name}.{value_name}": value + for value_name, value in _bckc.items() + } # prepend model name to bckc self.bckc = dict(self.bckc, **_model_bckc) return @@ -99,12 +116,16 @@ def _ln_likelihood(self): self.data[key].sel(t=self.plasma.time_to_calculate).astype("float128") ) - exp_error = exp_data * self.percent_error # Assume percentage error if none given. + exp_error = ( + exp_data * self.percent_error + ) # Assume percentage error if none given. if hasattr(self.data[key], "error"): if ( self.data[key].error != 0 ).any(): # TODO: Some models have an error of 0 given - exp_error = self.data[key].error.sel(t=self.plasma.time_to_calculate) + exp_error = self.data[key].error.sel( + t=self.plasma.time_to_calculate + ) _ln_likelihood = np.log(gaussian(model_data, exp_data, exp_error)) # treat channel as key dim which isn't averaged like other dims @@ -171,10 +192,10 @@ def sample_from_priors(self, param_names, size=10): samples = samples[:, 0:size] return samples.transpose() - def sample_from_high_density_region(self, param_names: list, sampler: object, nwalkers: int, nsamples = 100): - start_points = self.sample_from_priors( - param_names, size=nsamples - ) + def sample_from_high_density_region( + self, param_names: list, sampler: object, nwalkers: int, nsamples=100 + ): + start_points = self.sample_from_priors(param_names, size=nsamples) ln_prob, _ = sampler.compute_log_prob(start_points) num_best_points = int(nsamples * 0.05) @@ -190,17 +211,13 @@ def sample_from_high_density_region(self, param_names: list, sampler: object, nw best_points_std * 2, size=(nwalkers * 5, len(param_names)), ) - start = { - name: sample[:, idx] for idx, name in enumerate(param_names) - } + start = {name: sample[:, idx] for idx, name in enumerate(param_names)} ln_prior = self._ln_prior(start) # Convert from dictionary of arrays -> array, # then filtering out where ln_prior is -infinity - accepted_samples = np.array(list(start.values()))[ - :, ln_prior != -np.inf - ] + accepted_samples = np.array(list(start.values()))[:, ln_prior != -np.inf] samples = np.append(samples, accepted_samples, axis=1) - start_points = samples[:, 0: nwalkers].transpose() + start_points = samples[:, 0:nwalkers].transpose() return start_points def ln_posterior(self, parameters: dict, **kwargs): @@ -234,7 +251,9 @@ def ln_posterior(self, parameters: dict, **kwargs): kin_profs = {} for profile_key in PROFILES: if hasattr(self.plasma, profile_key): - kin_profs[profile_key] = getattr(self.plasma, profile_key).sel(t=self.plasma.time_to_calculate) + kin_profs[profile_key] = getattr(self.plasma, profile_key).sel( + t=self.plasma.time_to_calculate + ) else: raise ValueError(f"plasma does not have attribute {profile_key}") diff --git a/indica/models/charge_exchange.py b/indica/models/charge_exchange.py index 1aae94ac..1f0eeda8 100644 --- a/indica/models/charge_exchange.py +++ b/indica/models/charge_exchange.py @@ -22,7 +22,6 @@ def __init__( element: str = "c", instrument_method="get_charge_exchange", ): - self.name = name self.element = element self.instrument_method = instrument_method @@ -106,7 +105,8 @@ def __call__( return self.bckc -def pi_transform_example(nchannels:int): + +def pi_transform_example(nchannels: int): x_positions = np.linspace(0.2, 0.8, nchannels) y_positions = np.linspace(0.0, 0.0, nchannels) z_positions = np.linspace(0.0, 0.0, nchannels) @@ -120,13 +120,13 @@ def pi_transform_example(nchannels:int): ) return transect_transform + def example_run( pulse: int = None, diagnostic_name: str = "cxrs", plasma=None, plot=False, ): - # TODO: LOS sometimes crossing bad EFIT reconstruction if plasma is None: diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index d4538b20..12642af3 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -686,7 +686,6 @@ def __call__( self.intensity = self._calculate_intensity() self.spectra = self._make_spectra() - self._calculate_los_integral( calc_rho=calc_rho, ) diff --git a/indica/models/interferometry.py b/indica/models/interferometry.py index 29a6b42b..49cf391b 100644 --- a/indica/models/interferometry.py +++ b/indica/models/interferometry.py @@ -24,7 +24,6 @@ def __init__( name: str, instrument_method="get_interferometry", ): - self.name = name self.instrument_method = instrument_method self.quantities = AVAILABLE_QUANTITIES[self.instrument_method] @@ -88,6 +87,7 @@ def __call__( return self.bckc + def smmh1_transform_example(): los_start = np.array([[0.8, 0, 0], [0.8, 0, -0.1], [0.8, 0, -0.2]]) los_end = np.array([[0.17, 0, 0], [0.17, 0, -0.25], [0.17, 0, -0.2]]) @@ -106,6 +106,7 @@ def smmh1_transform_example(): ) return los_transform + def example_run(pulse: int = None, plasma=None, plot=False): if plasma is None: plasma = example_plasma(pulse=pulse) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 4ce7bac6..1715dd3f 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -462,7 +462,9 @@ def assign_profiles(self, impurity_to_profile=None): self.electron_density.loc[dict(t=t)] = self.Ne_prof() self.calc_impurity_density(self.impurities) self.electron_temperature.loc[dict(t=t)] = self.Te_prof() - self.ion_temperature.loc[dict(t=t)] = self.Ti_prof(y0_ref=self.Te_prof.y0) # Temp default Ti_ref + self.ion_temperature.loc[dict(t=t)] = self.Ti_prof( + y0_ref=self.Te_prof.y0 + ) # Temp default Ti_ref self.toroidal_rotation.loc[dict(t=t)] = self.Vrot_prof() self.neutral_density.loc[dict(t=t)] = self.Nh_prof() if ( @@ -1281,7 +1283,9 @@ def example_run( plasma.Nimp_prof.peaking = nimp_peaking[i] plasma.Nimp_prof.y0 = nimp_y0[i] plasma.Nimp_prof.wcenter = nimp_wcenter[i] - plasma.assign_profiles(impurity_to_profile="ar",) + plasma.assign_profiles( + impurity_to_profile="ar", + ) if pulse is None: equilibrium_data = fake_equilibrium_data( diff --git a/indica/readers/read_st40.py b/indica/readers/read_st40.py index 07e95aec..b23e25bd 100644 --- a/indica/readers/read_st40.py +++ b/indica/readers/read_st40.py @@ -153,7 +153,6 @@ def map_diagnostics(self, instruments: list, map_raw: bool = False): break def filter_data(self, instruments: list): - if not hasattr(self, "binned_data"): raise ValueError( "Bin data before filtering. No action permitted on raw data structure!" @@ -267,7 +266,6 @@ def __call__( map_diagnostics: bool = False, debug: bool = False, ): - if instruments is None: instruments = list(REVISIONS.keys()) if revisions is None: diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 215742ff..86057daa 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -8,16 +8,17 @@ from indica.readers.read_st40 import ReadST40 from indica.equilibrium import fake_equilibrium + class AbstractBayesWorkflow(ABC): @abstractmethod - def __init__(self, - phantoms = None, - diagnostics = None, - param_names=None, - opt_quantity=None, - priors=None, - ): - + def __init__( + self, + phantoms=None, + diagnostics=None, + param_names=None, + opt_quantity=None, + priors=None, + ): self.phantoms = phantoms self.diagnostics = diagnostics self.param_names = param_names @@ -31,10 +32,14 @@ def __init__(self, self.setup_opt_data(self.phantoms) self.setup_optimiser() - def read_test_data(self, diagnostic_transforms:dict): + def read_test_data(self, diagnostic_transforms: dict): # Used with phantom data for purposes of tests self.reader = None - self.equilibrium = fake_equilibrium(self.tstart, self.tend, self.dt, ) + self.equilibrium = fake_equilibrium( + self.tstart, + self.tend, + self.dt, + ) self.transforms = diagnostic_transforms self.data = {} @@ -92,21 +97,34 @@ def setup_optimiser(self, model_kwargs): def save_phantom_profiles(self, kinetic_profiles=None): if kinetic_profiles is None: - kinetic_profiles = ["electron_density", "impurity_density", "electron_temperature", - "ion_temperature", "ion_density", "fast_density", "neutral_density"] + kinetic_profiles = [ + "electron_density", + "impurity_density", + "electron_temperature", + "ion_temperature", + "ion_density", + "fast_density", + "neutral_density", + ] if self.phantoms: - phantom_profiles = {profile_key: getattr(self.plasma, profile_key).sel( - t=self.plasma.time_to_calculate).copy() - for profile_key in kinetic_profiles} + phantom_profiles = { + profile_key: getattr(self.plasma, profile_key) + .sel(t=self.plasma.time_to_calculate) + .copy() + for profile_key in kinetic_profiles + } else: - phantom_profiles = {profile_key: getattr(self.plasma, profile_key).sel( - t=self.plasma.time_to_calculate) * 0 - for profile_key in kinetic_profiles} + phantom_profiles = { + profile_key: getattr(self.plasma, profile_key).sel( + t=self.plasma.time_to_calculate + ) + * 0 + for profile_key in kinetic_profiles + } self.phantom_profiles = phantom_profiles def _build_result_dict(self): - """ Returns @@ -132,89 +150,134 @@ def _build_result_dict(self): "BURN_FRAC": self.burn_frac, "ITER": self.iterations, "NWALKERS": self.nwalkers, - "MODEL_KWARGS": self.model_kwargs, "OPT_QUANTITY": self.opt_quantity, "PARAM_NAMES": self.param_names, - "PULSE": self.pulse, "DT": self.dt, - "IMPURITIES":self.plasma.impurities, + "IMPURITIES": self.plasma.impurities, "MAIN_ION": self.plasma.main_ion, } result["INPUT"]["WORKFLOW"] = { - diag_name.upper(): - {"PULSE": self.pulse, # Change this if different pulses used for diagnostics - "USAGE": "".join([quantity[1] for quantity in quant_list if quantity[0] == diag_name]), - "RUN": "PLACEHOLDER", - } + diag_name.upper(): { + "PULSE": self.pulse, # Change this if different pulses used for diagnostics + "USAGE": "".join( + [quantity[1] for quantity in quant_list if quantity[0] == diag_name] + ), + "RUN": "PLACEHOLDER", + } for diag_name in self.diagnostics } - result["MODEL_DATA"] = {diag_name.upper(): - {quantity[1].upper(): self.blobs[f"{quantity[0]}.{quantity[1]}"] - for quantity in quant_list if quantity[0] == diag_name} - for diag_name in self.diagnostics - } + result["MODEL_DATA"] = { + diag_name.upper(): { + quantity[1].upper(): self.blobs[f"{quantity[0]}.{quantity[1]}"] + for quantity in quant_list + if quantity[0] == diag_name + } + for diag_name in self.diagnostics + } result["MODEL_DATA"]["SAMPLES"] = self.samples - result["DIAG_DATA"] = {diag_name.upper(): - {quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] - for quantity in quant_list if quantity[0] == diag_name} - for diag_name in self.diagnostics - } - + result["DIAG_DATA"] = { + diag_name.upper(): { + quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] + for quantity in quant_list + if quantity[0] == diag_name + } + for diag_name in self.diagnostics + } result["PHANTOMS"] = { "FLAG": self.phantoms, "NE": self.phantom_profiles["electron_density"], "TE": self.phantom_profiles["electron_temperature"], - "TI": self.phantom_profiles["ion_temperature"].sel(element=self.plasma.main_ion), - "NI": self.phantom_profiles["ion_density"].sel(element=self.plasma.main_ion), + "TI": self.phantom_profiles["ion_temperature"].sel( + element=self.plasma.main_ion + ), + "NI": self.phantom_profiles["ion_density"].sel( + element=self.plasma.main_ion + ), "NNEUTR": self.phantom_profiles["neutral_density"], "NFAST": self.phantom_profiles["fast_density"], } - result["PHANTOMS"].update({ - f"NIZ{num_imp+1}": self.phantom_profiles["impurity_density"].sel(element=imp) - for num_imp, imp in enumerate(self.plasma.impurities) - }) - result["PHANTOMS"].update({ - f"TIZ{num_imp + 1}": self.phantom_profiles["ion_temperature"].sel(element=imp) - for num_imp, imp in enumerate(self.plasma.impurities) - }) + result["PHANTOMS"].update( + { + f"NIZ{num_imp+1}": self.phantom_profiles["impurity_density"].sel( + element=imp + ) + for num_imp, imp in enumerate(self.plasma.impurities) + } + ) + result["PHANTOMS"].update( + { + f"TIZ{num_imp + 1}": self.phantom_profiles["ion_temperature"].sel( + element=imp + ) + for num_imp, imp in enumerate(self.plasma.impurities) + } + ) result["PROFILES"] = { "RHO_POLOIDAL": self.plasma.rho, "RHO_TOR": self.plasma.equilibrium.rhotor.interp(t=self.plasma.t), - "NE": self.blobs["electron_density"].median(dim="index"), - "NI": self.blobs["ion_density"].sel(element=self.plasma.main_ion).median(dim="index"), + "NI": self.blobs["ion_density"] + .sel(element=self.plasma.main_ion) + .median(dim="index"), "TE": self.blobs["electron_temperature"].median(dim="index"), - "TI": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion).median(dim="index"), + "TI": self.blobs["ion_temperature"] + .sel(element=self.plasma.main_ion) + .median(dim="index"), "NFAST": self.blobs["fast_density"].median(dim="index"), "NNEUTR": self.blobs["neutral_density"].median(dim="index"), - "NE_ERR": self.blobs["electron_density"].std(dim="index"), - "NI_ERR": self.blobs["ion_density"].sel(element=self.plasma.main_ion).std(dim="index"), + "NI_ERR": self.blobs["ion_density"] + .sel(element=self.plasma.main_ion) + .std(dim="index"), "TE_ERR": self.blobs["electron_temperature"].std(dim="index"), - "TI_ERR": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion).std(dim="index"), + "TI_ERR": self.blobs["ion_temperature"] + .sel(element=self.plasma.main_ion) + .std(dim="index"), "NFAST_ERR": self.blobs["fast_density"].std(dim="index"), "NNEUTR_ERR": self.blobs["neutral_density"].std(dim="index"), - } - result["PROFILES"] = {**result["PROFILES"], **{ - f"NIZ{num_imp+1}": self.blobs["impurity_density"].sel(element=imp).median(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities)}} - result["PROFILES"] = {**result["PROFILES"], **{ - f"NIZ{num_imp+1}_ERR": self.blobs["impurity_density"].sel(element=imp).std(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities)}} - result["PROFILES"] = {**result["PROFILES"], **{ - f"TIZ{num_imp + 1}": self.blobs["ion_temperature"].sel(element=imp).median(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities)}} - result["PROFILES"] = {**result["PROFILES"], **{ - f"TIZ{num_imp + 1}_ERR": self.blobs["ion_temperature"].sel(element=imp).std(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities)}} - + result["PROFILES"] = { + **result["PROFILES"], + **{ + f"NIZ{num_imp+1}": self.blobs["impurity_density"] + .sel(element=imp) + .median(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities) + }, + } + result["PROFILES"] = { + **result["PROFILES"], + **{ + f"NIZ{num_imp+1}_ERR": self.blobs["impurity_density"] + .sel(element=imp) + .std(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities) + }, + } + result["PROFILES"] = { + **result["PROFILES"], + **{ + f"TIZ{num_imp + 1}": self.blobs["ion_temperature"] + .sel(element=imp) + .median(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities) + }, + } + result["PROFILES"] = { + **result["PROFILES"], + **{ + f"TIZ{num_imp + 1}_ERR": self.blobs["ion_temperature"] + .sel(element=imp) + .std(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities) + }, + } result["PROFILE_STAT"] = { "SAMPLES": self.samples, @@ -226,13 +289,20 @@ def _build_result_dict(self): "NFAST": self.blobs["fast_density"], "NNEUTR": self.blobs["neutral_density"], } - result["PROFILE_STAT"] = {**result["PROFILE_STAT"], **{ - f"NIZ{num_imp+1}": self.blobs["impurity_density"].sel(element=imp) - for num_imp, imp in enumerate(self.plasma.impurities)}} - result["PROFILE_STAT"] = {**result["PROFILE_STAT"], **{ - f"TIZ{num_imp + 1}": self.blobs["ion_temperature"].sel(element=imp) - for num_imp, imp in enumerate(self.plasma.impurities)}} - + result["PROFILE_STAT"] = { + **result["PROFILE_STAT"], + **{ + f"NIZ{num_imp+1}": self.blobs["impurity_density"].sel(element=imp) + for num_imp, imp in enumerate(self.plasma.impurities) + }, + } + result["PROFILE_STAT"] = { + **result["PROFILE_STAT"], + **{ + f"TIZ{num_imp + 1}": self.blobs["ion_temperature"].sel(element=imp) + for num_imp, imp in enumerate(self.plasma.impurities) + }, + } result["OPTIMISATION"] = { "ACCEPT_FRAC": self.accept_frac, @@ -242,37 +312,80 @@ def _build_result_dict(self): } result["GLOBAL"] = { - "TI0": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion).sel(rho_poloidal=0, method="nearest").median(dim="index"), - "TE0": self.blobs["electron_temperature"].sel(rho_poloidal=0, method="nearest").median(dim="index"), - "NE0": self.blobs["electron_density"].sel(rho_poloidal=0, method="nearest").median(dim="index"), - "NI0": self.blobs["ion_density"].sel(element=self.plasma.main_ion).sel(rho_poloidal=0, method="nearest").median(dim="index"), - "TI0_ERR": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion).sel(rho_poloidal=0, method="nearest").std(dim="index"), - "TE0_ERR": self.blobs["electron_temperature"].sel(rho_poloidal=0, method="nearest").std(dim="index"), - "NE0_ERR": self.blobs["electron_density"].sel(rho_poloidal=0, method="nearest").std(dim="index"), - "NI0_ERR": self.blobs["ion_density"].sel(element=self.plasma.main_ion).sel(rho_poloidal=0, method="nearest").std(dim="index"), + "TI0": self.blobs["ion_temperature"] + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), + "TE0": self.blobs["electron_temperature"] + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), + "NE0": self.blobs["electron_density"] + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), + "NI0": self.blobs["ion_density"] + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), + "TI0_ERR": self.blobs["ion_temperature"] + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), + "TE0_ERR": self.blobs["electron_temperature"] + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), + "NE0_ERR": self.blobs["electron_density"] + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), + "NI0_ERR": self.blobs["ion_density"] + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), + } + result["GLOBAL"] = { + **result["GLOBAL"], + **{ + f"TI0Z{num_imp + 1}": self.blobs["ion_temperature"] + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities) + }, + } + result["GLOBAL"] = { + **result["GLOBAL"], + **{ + f"TI0Z{num_imp + 1}_ERR": self.blobs["ion_temperature"] + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities) + }, + } + result["GLOBAL"] = { + **result["GLOBAL"], + **{ + f"NI0Z{num_imp + 1}": self.blobs["impurity_density"] + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities) + }, + } + result["GLOBAL"] = { + **result["GLOBAL"], + **{ + f"NI0Z{num_imp + 1}_ERR": self.blobs["impurity_density"] + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index") + for num_imp, imp in enumerate(self.plasma.impurities) + }, } - result["GLOBAL"] = {**result["GLOBAL"], **{ - f"TI0Z{num_imp + 1}": self.blobs["ion_temperature"].sel(element=imp).sel(rho_poloidal=0, method="nearest").median(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities)}} - result["GLOBAL"] = {**result["GLOBAL"], **{ - f"TI0Z{num_imp + 1}_ERR": self.blobs["ion_temperature"].sel(element=imp).sel(rho_poloidal=0, - method="nearest").std( - dim="index") - for num_imp, imp in enumerate(self.plasma.impurities)}} - result["GLOBAL"] = {**result["GLOBAL"], **{ - f"NI0Z{num_imp + 1}": self.blobs["impurity_density"].sel(element=imp).sel(rho_poloidal=0, method="nearest").median(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities)}} - result["GLOBAL"] = {**result["GLOBAL"], **{ - f"NI0Z{num_imp + 1}_ERR": self.blobs["impurity_density"].sel(element=imp).sel(rho_poloidal=0, - method="nearest").std( - dim="index") - for num_imp, imp in enumerate(self.plasma.impurities)}} self.result = result return self.result def run_sampler(self): - """ TODO: unsure if keeping in abstract class is best practice @@ -284,7 +397,11 @@ def run_sampler(self): """ self.autocorr = sample_with_autocorr( - self.sampler, self.start_points, self.iterations, self.param_names.__len__(), auto_sample=10 + self.sampler, + self.start_points, + self.iterations, + self.param_names.__len__(), + auto_sample=10, ) blobs = self.sampler.get_blobs( discard=int(self.iterations * self.burn_frac), flat=True @@ -306,7 +423,6 @@ def run_sampler(self): self.post_sample = self.sampler.get_chain(flat=True) self.result = self._build_result_dict() - def save_pickle(self, filepath): if filepath: Path(filepath).mkdir(parents=True, exist_ok=True) @@ -345,11 +461,11 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl if sampler.iteration % auto_sample: continue new_tau = sampler.get_autocorr_time(tol=0) - autocorr[sampler.iteration - 1, ] = new_tau + autocorr[sampler.iteration - 1,] = new_tau converged = np.all(new_tau * 50 < sampler.iteration) converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) if converged: break old_tau = new_tau - autocorr = autocorr[: sampler.iteration, ] - return autocorr \ No newline at end of file + autocorr = autocorr[: sampler.iteration,] + return autocorr diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index a4e06773..8695969d 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -176,7 +176,7 @@ def violinplot( axs.set_xlabel(key) top = axs.get_ylim()[1] bot = axs.get_ylim()[0] - axs.set_ylim( top=top * 1.1, bottom=bot * 0.9) + axs.set_ylim(top=top * 1.1, bottom=bot * 0.9) axs.set_ylabel(f"{ylabel}") y = diag_data[key].sel(t=data[key].t).values axs.errorbar( @@ -218,17 +218,14 @@ def histograms(data, diag_data, filename): plt.close() - def plot_autocorr(autocorr, param_names, figheader, filetype=".png"): - plt.figure() - x_data = np.ones(shape=(autocorr.shape)) * np.arange(0, autocorr[:,0].__len__())[:,None] - plt.plot( - np.where(np.isfinite(autocorr), x_data, np.nan), - autocorr, - "x" + x_data = ( + np.ones(shape=(autocorr.shape)) + * np.arange(0, autocorr[:, 0].__len__())[:, None] ) + plt.plot(np.where(np.isfinite(autocorr), x_data, np.nan), autocorr, "x") plt.legend(param_names) plt.xlabel("iterations") plt.ylabel("auto-correlation time (iterations)") @@ -242,7 +239,6 @@ def plot_bayes_result( filetype=".png", **kwargs, ): - # delete all but pickle in directory for root, dirs, files in os.walk(figheader): for f in files: @@ -380,7 +376,7 @@ def plot_bayes_result( filetype=filetype, phantom_profile=phantom_profiles, color="blue", - sharefig=True + sharefig=True, ) key = "NI" plot_profile( diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index b2d5f4e3..02b20ec1 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -9,7 +9,10 @@ from indica.writers.bda_tree import create_nodes, write_nodes, check_analysis_run from indica.models.interferometry import Interferometry, smmh1_transform_example -from indica.models.helike_spectroscopy import HelikeSpectrometer, helike_transform_example +from indica.models.helike_spectroscopy import ( + HelikeSpectrometer, + helike_transform_example, +) from indica.models.charge_exchange import ChargeExchange, pi_transform_example from indica.models.equilibrium_reconstruction import EquilibriumReconstruction from indica.models.plasma import Plasma @@ -102,29 +105,27 @@ class BayesWorkflowExample(AbstractBayesWorkflow): def __init__( - self, - pulse: int = None, - pulse_to_write: bool = None, - run: str = "RUN01", - diagnostics: list = None, - param_names: list = None, - opt_quantity: list = None, - priors: dict = None, - profile_params: dict = None, - phantoms: bool = False, - model_kwargs: dict = {}, - - nwalkers: int = 50, - tstart: float = 0.02, - tend: float = 0.10, - dt: float = 0.01, - tsample: float = 0.06, - iterations: int = 100, - burn_frac: float = 0, - - mds_write: bool = False, - plot: bool = True, - sample_high_density: bool = False, + self, + pulse: int = None, + pulse_to_write: bool = None, + run: str = "RUN01", + diagnostics: list = None, + param_names: list = None, + opt_quantity: list = None, + priors: dict = None, + profile_params: dict = None, + phantoms: bool = False, + model_kwargs: dict = {}, + nwalkers: int = 50, + tstart: float = 0.02, + tend: float = 0.10, + dt: float = 0.01, + tsample: float = 0.06, + iterations: int = 100, + burn_frac: float = 0, + mds_write: bool = False, + plot: bool = True, + sample_high_density: bool = False, ): self.pulse = pulse self.pulse_to_write = pulse_to_write @@ -149,18 +150,30 @@ def __init__( self.plot = plot self.sample_high_density = sample_high_density - for attribute in ["param_names", "opt_quantity", "priors", "diagnostics", "profile_params"]: + for attribute in [ + "param_names", + "opt_quantity", + "priors", + "diagnostics", + "profile_params", + ]: if getattr(self, attribute) is None: raise ValueError(f"{attribute} needs to be defined") if self.pulse is None and self.phantoms is False: - raise ValueError("Set phantoms to True when running phantom plasma i.e. pulse=None") + raise ValueError( + "Set phantoms to True when running phantom plasma i.e. pulse=None" + ) if pulse is None: print("Running in test mode") - self.read_test_data({"xrcs": helike_transform_example(1), - "smmh1": smmh1_transform_example(), - "cxff_pi": pi_transform_example(5)}) + self.read_test_data( + { + "xrcs": helike_transform_example(1), + "smmh1": smmh1_transform_example(), + "cxff_pi": pi_transform_example(5), + } + ) else: self.read_data(self.diagnostics) self.setup_plasma() @@ -220,12 +233,14 @@ def setup_models(self, diagnostics: list): window_vector = None if hasattr(self, "data"): if diag in self.data.keys(): - window_vector = self.data[diag]["spectra"].wavelength.values * 0.1 + window_vector = ( + self.data[diag]["spectra"].wavelength.values * 0.1 + ) model = HelikeSpectrometer( name="xrcs", window_masks=[slice(0.394, 0.396)], - window_vector=window_vector + window_vector=window_vector, ) model.set_los_transform(los_transform) @@ -253,46 +268,42 @@ def _phantom_data(self, noise=False, noise_factor=0.1): if "smmh1" in self.diagnostics: opt_data["smmh1.ne"] = ( self.models["smmh1"]() - .pop("ne") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("ne") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if "xrcs" in self.diagnostics: opt_data["xrcs.spectra"] = ( self.models["xrcs"]() - .pop("spectra") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - ) - opt_data["xrcs.spectra"]["error"] = np.sqrt( - opt_data["xrcs.spectra"] + .pop("spectra") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) + opt_data["xrcs.spectra"]["error"] = np.sqrt(opt_data["xrcs.spectra"]) if "cxff_pi" in self.diagnostics: cxrs_data = ( self.models["cxff_pi"]() - .pop("ti") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("ti") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2, drop=True) if "efit" in self.diagnostics: opt_data["efit.wp"] = ( self.models["efit"]() - .pop("wp") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("wp") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if noise: opt_data["smmh1.ne"] = opt_data["smmh1.ne"] + self.opt_data[ "smmh1.ne" ].max().values * np.random.normal(0, noise_factor, None) - opt_data["xrcs.spectra"] = opt_data[ - "xrcs.spectra" - ] + np.random.normal( + opt_data["xrcs.spectra"] = opt_data["xrcs.spectra"] + np.random.normal( 0, np.sqrt(opt_data["xrcs.spectra"].values[0,]), opt_data["xrcs.spectra"].shape[1], ) - opt_data["cxff_pi.ti"] = opt_data[ - "cxff_pi.ti" - ] + opt_data["cxff_pi.ti"].max().values * np.random.normal( + opt_data["cxff_pi.ti"] = opt_data["cxff_pi.ti"] + opt_data[ + "cxff_pi.ti" + ].max().values * np.random.normal( 0, noise_factor, opt_data["cxff_pi.ti"].shape[1] ) opt_data["efit.wp"] = opt_data["efit.wp"] + opt_data[ @@ -304,15 +315,16 @@ def _exp_data(self): opt_data = flatdict.FlatDict(self.data, ".") if "xrcs" in self.diagnostics: opt_data["xrcs.spectra"]["wavelength"] = ( - opt_data["xrcs.spectra"].wavelength * 0.1 + opt_data["xrcs.spectra"].wavelength * 0.1 ) background = opt_data["xrcs.spectra"].where( (opt_data["xrcs.spectra"].wavelength < 0.392) & (opt_data["xrcs.spectra"].wavelength > 0.388), drop=True, ) - self.model_kwargs["xrcs_background"] = background.mean(dim="wavelength").sel( - t=self.plasma.time_to_calculate) + self.model_kwargs["xrcs_background"] = background.mean( + dim="wavelength" + ).sel(t=self.plasma.time_to_calculate) opt_data["xrcs.spectra"]["error"] = np.sqrt( opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 ) @@ -324,7 +336,6 @@ def _exp_data(self): return opt_data def setup_optimiser(self, model_kwargs): - self.bayesmodel = BayesModels( plasma=self.plasma, data=self.opt_data, @@ -343,12 +354,15 @@ def setup_optimiser(self, model_kwargs): moves=self.move, kwargs=model_kwargs, ) - self.start_points = self._sample_start_points(sample_high_density=self.sample_high_density) + self.start_points = self._sample_start_points( + sample_high_density=self.sample_high_density + ) def _sample_start_points(self, sample_high_density: bool = True): if sample_high_density: - start_points = self.bayesmodel.sample_from_high_density_region(self.param_names, self.sampler, - self.nwalkers) + start_points = self.bayesmodel.sample_from_high_density_region( + self.param_names, self.sampler, self.nwalkers + ) else: start_points = self.bayesmodel.sample_from_priors( self.param_names, size=self.nwalkers @@ -356,12 +370,13 @@ def _sample_start_points(self, sample_high_density: bool = True): return start_points def __call__(self, filepath="./results/test/", **kwargs): - if self.mds_write: # check_analysis_run(self.pulse, self.run) - self.node_structure = create_nodes(pulse_to_write=self.pulse_to_write, - diagnostic_quantities=self.opt_quantity, - mode="NEW") + self.node_structure = create_nodes( + pulse_to_write=self.pulse_to_write, + diagnostic_quantities=self.opt_quantity, + mode="NEW", + ) self.run_sampler() self.save_pickle(filepath=filepath) @@ -386,17 +401,19 @@ def __call__(self, filepath="./results/test/", **kwargs): param_names=OPTIMISED_PARAMS, profile_params=DEFAULT_PROFILE_PARAMS, priors=DEFAULT_PRIORS, - model_kwargs={"xrcs_moment_analysis": False, }, + model_kwargs={ + "xrcs_moment_analysis": False, + }, phantoms=True, - iterations=200, nwalkers=50, burn_frac=0.10, dt=0.005, tsample=0.060, - mds_write=True, plot=True, sample_high_density=False, ) - results = run(filepath="./results/test/", ) + results = run( + filepath="./results/test/", + ) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 1d16083c..6f25ecb7 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -10,35 +10,29 @@ def bda(): nodes = { "TIME": ("NUMERIC", "time vector, s"), "TIME_OPT": ("NUMERIC", "time of optimisation, s"), - "INPUT": { "BURN_FRAC": ("NUMERIC", "Burn in fraction for chains"), "ITER": ("NUMERIC", "Iterations of optimiser"), "PARAM_NAMES": ("TEXT", "Names of parameters optimised"), - "OPT_QUANTITY":("TEXT", "Names of quantities optimised"), + "OPT_QUANTITY": ("TEXT", "Names of quantities optimised"), "MODEL_KWARGS": ("TEXT", "Model key word arguments"), # "OPT_KWARGS": ("TEXT", "Optimiser key word arguments"), "PULSE": ("NUMERIC", "Pulse number"), - "TSTART": ("NUMERIC", "Start of time vector, s"), "TEND": ("NUMERIC", "End of time vector, s"), "DT": ("NUMERIC", "Distance between time points, s"), "TSAMPLE": ("NUMERIC", "Sample time, s"), "IMPURITIES": ("TEXT", "Names of impurity elements"), "MAIN_ION": ("TEXT", "Name of main ion"), - }, - "METADATA": { "GITCOMMIT": ("TEXT", "Commit ID used for run"), "USER": ("TEXT", "Username of owner"), "EQUIL": ("TEXT", "Equilibrium used"), }, - "PROFILES": { "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), "RHOT": ("SIGNAL", "Radial vector, toroidal flux"), - "NE": ("SIGNAL", "Electron density, m^-3"), "NI": ("SIGNAL", "Ion density, m^-3"), "TE": ("SIGNAL", "Electron temperature, eV"), @@ -51,7 +45,6 @@ def bda(): "NIZ3": ("SIGNAL", "Density of impurity Z3, m^-3"), "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), "NFAST": ("SIGNAL", "Density of fast ion, m^-3"), - "ZI": ("SIGNAL", "Average charge of main ion, "), "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), @@ -59,7 +52,6 @@ def bda(): "ZEFF": ("SIGNAL", "Effective charge, "), "P": ("SIGNAL", "Pressure,Pa"), "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), - "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), @@ -72,9 +64,7 @@ def bda(): "NIZ3_ERR": ("SIGNAL", "Density of impurity Z3 error, m^-3"), "NNEUTR_ERR": ("SIGNAL", "Density of neutral main ion error, m^-3"), "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), - }, - "PROFILE_STAT": { "SAMPLES": ("NUMERIC", "Numerical index of the optimisation samples"), "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), @@ -90,7 +80,6 @@ def bda(): "NIZ3": ("SIGNAL", "Density of impurity IMP3, m^-3"), "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), "NFAST": ("SIGNAL", "Density of fast ions, m^-3"), - "ZI": ("SIGNAL", "Average charge of main ion, "), "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), @@ -99,7 +88,6 @@ def bda(): "P": ("SIGNAL", "Pressure,Pa"), "VOLUME": ("SIGNAL", "Volume inside magnetic surface, m^3"), }, - "GLOBAL": { "NE0": ("SIGNAL", "Central electron density, m^-3"), "NI0": ("SIGNAL", "Central ion density, m^-3"), @@ -111,7 +99,6 @@ def bda(): "NI0Z1": ("SIGNAL", "Central density of impurity Z1, m^-3"), "NI0Z2": ("SIGNAL", "Central density of impurity Z2, m^-3"), "NI0Z3": ("SIGNAL", "Central density of impurity Z3, m^-3"), - "NE0_ERR": ("SIGNAL", "Central electron density error, m^-3"), "NI0_ERR": ("SIGNAL", "Central ion density error, m^-3"), "TE0_ERR": ("SIGNAL", "Central electron temperature error, eV"), @@ -122,11 +109,13 @@ def bda(): "NI0Z1_ERR": ("SIGNAL", "Central density of impurity Z1, m^-3"), "NI0Z2_ERR": ("SIGNAL", "Central density of impurity Z2, m^-3"), "NI0Z3_ERR": ("SIGNAL", "Central density of impurity Z3, m^-3"), - }, "PHANTOMS": { "FLAG": ("TEXT", "True if phantoms used"), - "RHO_POLOIDAL": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "RHO_POLOIDAL": ( + "NUMERIC", + "Radial vector, Sqrt of normalised poloidal flux", + ), "NE": ("SIGNAL", "Electron density, m^-3"), "NI": ("SIGNAL", "Ion density, m^-3"), "TE": ("SIGNAL", "Electron temperature, eV"), @@ -137,9 +126,7 @@ def bda(): "NIZ1": ("SIGNAL", "Impurity density of Z1, m^-3"), "NIZ2": ("SIGNAL", "Impurity density of Z2, m^-3"), "NIZ3": ("SIGNAL", "Impurity density of Z3, m^-3"), - }, - "OPTIMISATION": { "ACCEPT_FRAC": ("NUMERIC", "Fraction of samples accepted by optimiser"), "AUTO_CORR": ("NUMERIC", "Auto-correlation (iteration nwalker)"), @@ -150,37 +137,53 @@ def bda(): return nodes -DIAGNOSTIC_QUANTITY = ["DIAGNOSTIC1.QUANTITY1", "DIAGNOSTIC1.QUANTITY2", "DIAGNOSTIC2.QUANTITY1"] +DIAGNOSTIC_QUANTITY = [ + "DIAGNOSTIC1.QUANTITY1", + "DIAGNOSTIC1.QUANTITY2", + "DIAGNOSTIC2.QUANTITY1", +] -def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_quantities=DIAGNOSTIC_QUANTITY, mode="EDIT"): +def create_nodes( + pulse_to_write=23000101, + run="RUN01", + best=True, + diagnostic_quantities=DIAGNOSTIC_QUANTITY, + mode="EDIT", +): bda_nodes = bda() - quant_list = [item.upper().split(".") for item in diagnostic_quantities] # replace OPTIMISED_QUANTITY + quant_list = [ + item.upper().split(".") for item in diagnostic_quantities + ] # replace OPTIMISED_QUANTITY diag_names = list(set([item[0] for item in quant_list])) - diag_nodes = {diag_name: - {quantity[1]: ("SIGNAL", f"measured {quantity[1]} from {quantity[0]}") - for quantity in quant_list if quantity[0] == diag_name} - for diag_name in diag_names - } + diag_nodes = { + diag_name: { + quantity[1]: ("SIGNAL", f"measured {quantity[1]} from {quantity[0]}") + for quantity in quant_list + if quantity[0] == diag_name + } + for diag_name in diag_names + } # for diag_name in diag_names: # diag_nodes[diag_name]["RUN"] = ("TEXT", f"RUN from which {diag_name} data was taken") nodes = { "RUN": ("TEXT", "RUN used for diagnostic"), "USAGE": ("TEXT", "Quantity used in analysis"), - "PULSE": ("NUMERIC", "Pulse used for diagnostic") + "PULSE": ("NUMERIC", "Pulse used for diagnostic"), } - workflow_nodes = {diag_name: nodes - for diag_name in diag_names - } + workflow_nodes = {diag_name: nodes for diag_name in diag_names} - model_nodes = {diag_name: - {quantity[1]: ("SIGNAL", f"modelled {quantity[1]} from {quantity[0]}") - for quantity in quant_list if quantity[0] == diag_name} - for diag_name in diag_names - } + model_nodes = { + diag_name: { + quantity[1]: ("SIGNAL", f"modelled {quantity[1]} from {quantity[0]}") + for quantity in quant_list + if quantity[0] == diag_name + } + for diag_name in diag_names + } model_nodes["SAMPLES"] = ("NUMERIC", "index of samples taken from optimisation") bda_nodes["MODEL_DATA"] = model_nodes bda_nodes["DIAG_DATA"] = diag_nodes @@ -201,7 +204,7 @@ def create_nodes(pulse_to_write=23000101, run="RUN01", best=True, diagnostic_qua tree=tree, pulse_number=pulse_to_write, base_node_to_read=None, - node_information_file=bda_nodes + node_information_file=bda_nodes, ).get() util.StandardNodeCreation( @@ -224,26 +227,33 @@ def write_nodes(pulse, node_info, data): ) -def check_analysis_run(pulseNo, which_run, ): +def check_analysis_run( + pulseNo, + which_run, +): # Checker function to see if data already exists in a run - IP_address_smaug = '192.168.1.7:8000' + IP_address_smaug = "192.168.1.7:8000" conn = Connection(IP_address_smaug) - conn.openTree('BDA', pulseNo) + conn.openTree("BDA", pulseNo) - temp = conn.get('BDA.' + which_run + '.TIME').data() + temp = conn.get("BDA." + which_run + ".TIME").data() conn.closeAllTrees() overwrite_flag = True if isinstance(temp, np.ndarray): - print(f'Data already Exists in pulseNo = {pulseNo}, which_run = {which_run}') - print('User prompt...') - question = f' Scheduled to overwrite pulseNo {pulseNo}, {which_run}' \ - f'\n Do you want to overwrite {which_run}? (y/n)' + print(f"Data already Exists in pulseNo = {pulseNo}, which_run = {which_run}") + print("User prompt...") + question = ( + f" Scheduled to overwrite pulseNo {pulseNo}, {which_run}" + f"\n Do you want to overwrite {which_run}? (y/n)" + ) overwrite_flag = query_yes_no(question) return overwrite_flag -def query_yes_no(question, ): +def query_yes_no( + question, +): valid = {"yes": True, "y": True, "no": False, "n": False} while True: sys.stdout.write(question) diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index e4033134..2da073e9 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -2,8 +2,13 @@ import copy import numpy as np -from indica.workflows.bayes_workflow_example import BayesWorkflowExample, DEFAULT_PRIORS, DEFAULT_PROFILE_PARAMS, \ - OPTIMISED_PARAMS, OPTIMISED_QUANTITY +from indica.workflows.bayes_workflow_example import ( + BayesWorkflowExample, + DEFAULT_PRIORS, + DEFAULT_PROFILE_PARAMS, + OPTIMISED_PARAMS, + OPTIMISED_QUANTITY, +) """ TODO: @@ -16,27 +21,28 @@ class TestBayesWorkflowExample: def setup_class(self): - - self.default_settings = dict(pulse=None, - pulse_to_write=23000111, - run="RUN01", - diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], - opt_quantity=OPTIMISED_QUANTITY, - param_names=OPTIMISED_PARAMS, - profile_params=DEFAULT_PROFILE_PARAMS, - priors=DEFAULT_PRIORS, - model_kwargs={"xrcs_moment_analysis": False, }, - phantoms=True, - - iterations=1, - nwalkers=20, - burn_frac=0.10, - dt=0.005, - tsample=0.060, - - mds_write=False, - plot=False, - sample_high_density=False, ) + self.default_settings = dict( + pulse=None, + pulse_to_write=23000111, + run="RUN01", + diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], + opt_quantity=OPTIMISED_QUANTITY, + param_names=OPTIMISED_PARAMS, + profile_params=DEFAULT_PROFILE_PARAMS, + priors=DEFAULT_PRIORS, + model_kwargs={ + "xrcs_moment_analysis": False, + }, + phantoms=True, + iterations=1, + nwalkers=20, + burn_frac=0.10, + dt=0.005, + tsample=0.060, + mds_write=False, + plot=False, + sample_high_density=False, + ) self.untouched_workflow = BayesWorkflowExample(**self.default_settings) self.workflow = copy.deepcopy(self.untouched_workflow) @@ -48,7 +54,15 @@ def teardown_method(self): self.workflow = None def test_workflow_initializes(self): - attributes_to_check = ["plasma", "phantom_profiles", "data", "reader", "opt_data", "models", "bayesmodel",] + attributes_to_check = [ + "plasma", + "phantom_profiles", + "data", + "reader", + "opt_data", + "models", + "bayesmodel", + ] for attribute in attributes_to_check: if not hasattr(self.workflow, attribute): raise ValueError(f"missing {attribute} in workflow object") @@ -56,11 +70,15 @@ def test_workflow_initializes(self): def test_init_phantoms_false_with_example_plasma(self): with pytest.raises(ValueError): - example = BayesWorkflowExample(dict(self.default_settings, **{"phantoms": False})) + example = BayesWorkflowExample( + dict(self.default_settings, **{"phantoms": False}) + ) def test_init_not_including_all_required_inputs(self): with pytest.raises(ValueError): - example = BayesWorkflowExample(dict(self.default_settings, **{"param_names": None})) + example = BayesWorkflowExample( + dict(self.default_settings, **{"param_names": None}) + ) # def test_reader_has_read_all_diagnostic_data(self): # assert all(diag_name in self.workflow.reader.keys() for diag_name in self.workflow.diagnostics) @@ -71,7 +89,7 @@ def test_plasma_has_equilibrium(self): def test_phantom_profiles_are_not_mutatable(self): phantoms = copy.deepcopy(self.workflow.phantom_profiles) self.workflow.plasma.electron_temperature += 1 - assert (phantoms is not self.workflow.phantom_profiles) + assert phantoms is not self.workflow.phantom_profiles def test_setup_models_with_wrong_diagnostic_names(self): with pytest.raises(Exception): @@ -113,7 +131,7 @@ def test_worklow_has_results_after_run(self): raise ValueError(f"missing result in workflow object") assert True + if __name__ == "__main__": test = TestBayesWorkflowExample() test.setup_class() - From acc50b574fbf3337f2b9c59266705354e70761d6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 10 Aug 2023 10:44:00 +0100 Subject: [PATCH 120/288] renamed kin_prof -> plasma_profiles --- indica/bayesmodels.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index c8801664..dd9e5410 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -70,7 +70,6 @@ def __init__( def _build_bckc(self, params, **kwargs): """ - TODO: consider how to handle when models have overlapping kwargs Parameters ---------- params - dictionary which is updated by optimiser @@ -82,14 +81,14 @@ def _build_bckc(self, params, **kwargs): """ self.bckc: dict = {} for model in self.diagnostic_models: - # removes "model.name_" from params and kwargs then passes them to model e.g. xrcs_background -> background + # removes "model.name." from params and kwargs then passes them to model e.g. xrcs.background -> background _nuisance_params = { - param_name.replace(model.name + "_", ""): param_value + param_name.replace(model.name + ".", ""): param_value for param_name, param_value in params.items() if model.name in param_name } _model_settings = { - kwarg_name.replace(model.name + "_", ""): kwarg_value + kwarg_name.replace(model.name + ".", ""): kwarg_value for kwarg_name, kwarg_value in kwargs.items() if model.name in kwarg_name } @@ -248,14 +247,14 @@ def ln_posterior(self, parameters: dict, **kwargs): ln_likelihood = self._ln_likelihood() # compare results to data ln_posterior = ln_likelihood + ln_prior - kin_profs = {} + plasma_profiles = {} for profile_key in PROFILES: if hasattr(self.plasma, profile_key): - kin_profs[profile_key] = getattr(self.plasma, profile_key).sel( + plasma_profiles[profile_key] = getattr(self.plasma, profile_key).sel( t=self.plasma.time_to_calculate ) else: raise ValueError(f"plasma does not have attribute {profile_key}") - blob = deepcopy({**self.bckc, **kin_profs}) + blob = deepcopy({**self.bckc, **plasma_profiles}) return ln_posterior, blob From 223a526f67a3339934a3bc49de6c4e6af2e633bc Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 10 Aug 2023 10:47:57 +0100 Subject: [PATCH 121/288] refactored window handling --- indica/models/helike_spectroscopy.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 12642af3..db5c4d39 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -46,7 +46,7 @@ def __init__( element: str = "ar", window_len: int = 1030, window_lim: list = [0.394, 0.401], - window_vector=None, + window: np.array = None, window_masks: list = [], ): """ @@ -77,6 +77,8 @@ def __init__( self.full_run = full_run self.adf15 = ADF15 self.pecs: dict + self.window = window + self.window_masks = window_masks if self.marchuk: marchuck_reader = MARCHUKReader( @@ -87,16 +89,13 @@ def __init__( else: self.pecs = self._set_adas_pecs() - self.window_masks = window_masks - self.window_vector = window_vector - if self.window_vector is not None: - window = self.window_vector - else: - window = np.linspace(window_lim[0], window_lim[1], window_len) - mask = np.zeros(shape=window.shape) - if window_masks: - for mslice in window_masks: - mask[(window > mslice.start) & (window < mslice.stop)] = 1 + if self.window is None: + self.window = np.linspace(window_lim[0], window_lim[1], window_len) + + mask = np.zeros(shape=self.window.shape) + if self.window_masks: + for mslice in self.window_masks: + mask[(self.window > mslice.start) & (self.window < mslice.stop)] = 1 else: mask[:] = 1 From c9333ae75893d1ec581d52b1e4ad9a86eab94316 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 10 Aug 2023 15:11:39 +0100 Subject: [PATCH 122/288] replaced doppler_broaden with physics.ev_doppler --- indica/models/helike_spectroscopy.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index db5c4d39..083ca4e9 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -3,10 +3,10 @@ import matplotlib.cm as cm import matplotlib.pylab as plt import numpy as np -from scipy import constants import xarray as xr from xarray import DataArray +from indica.physics import ev_doppler from indica.converters.line_of_sight import LineOfSightTransform from indica.datatypes import ELEMENTS from indica.models.abstractdiagnostic import DiagnosticModel @@ -90,12 +90,12 @@ def __init__( self.pecs = self._set_adas_pecs() if self.window is None: - self.window = np.linspace(window_lim[0], window_lim[1], window_len) + window = np.linspace(window_lim[0], window_lim[1], window_len) - mask = np.zeros(shape=self.window.shape) + mask = np.zeros(shape=window.shape) if self.window_masks: for mslice in self.window_masks: - mask[(self.window > mslice.start) & (self.window < mslice.stop)] = 1 + mask[(window > mslice.start) & (window < mslice.stop)] = 1 else: mask[:] = 1 @@ -492,13 +492,8 @@ def _make_spectra( """ element = self.pecs["element"] - _spectra = doppler_broaden( - self.window[self.window > 0].wavelength, - self.intensity, - self.intensity.wavelength, - self.ion_mass, - self.Ti.sel(element=element), - ) + _sigma = ev_doppler(self.Ti.sel(element=element), self.ion_mass) * self.window.wavelength + _spectra = gaussian(self.window[self.window > 0].wavelength, self.intensity, self.intensity.wavelength, _sigma) _spectra = _spectra.sum("line_name") # extend spectra to same coords as self.window.wavelength with NaNs # to maintain same shape as mds data @@ -697,18 +692,9 @@ def __call__( return self.bckc -# fmt: off -def doppler_broaden(x, integral, center, ion_mass, ion_temp): - mass = (ion_mass * constants.proton_mass * constants.c ** 2) - sigma = np.sqrt(constants.e / mass * ion_temp) * center - gaussian_broadened = gaussian(x, integral, center, sigma, ) - return gaussian_broadened - - def gaussian(x, integral, center, sigma): return (integral / (sigma * np.sqrt(2 * np.pi)) * np.exp(-((x - center) ** 2) / (2 * sigma ** 2))) -# fmt: on def select_transition(adf15_data, transition: str, wavelength: float): From 80bc391c3f3b09c395bbac4eb09942dbd98a88c8 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 10 Aug 2023 15:13:30 +0100 Subject: [PATCH 123/288] removed methods from __init__ --- indica/workflows/abstract_bayes_workflow.py | 8 +- indica/workflows/bayes_workflow_example.py | 148 ++++++++++---------- 2 files changed, 76 insertions(+), 80 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 86057daa..d9ff7184 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -13,24 +13,21 @@ class AbstractBayesWorkflow(ABC): @abstractmethod def __init__( self, + pulse=None, phantoms=None, diagnostics=None, param_names=None, opt_quantity=None, priors=None, ): + self.pulse = pulse self.phantoms = phantoms self.diagnostics = diagnostics self.param_names = param_names self.opt_quantity = opt_quantity self.priors = priors - self.read_data(self.diagnostics) - self.setup_plasma() - self.save_phantom_profiles() self.setup_models(self.diagnostics) - self.setup_opt_data(self.phantoms) - self.setup_optimiser() def read_test_data(self, diagnostic_transforms: dict): # Used with phantom data for purposes of tests @@ -59,6 +56,7 @@ def setup_plasma(self): """ self.plasma = None self.plasma.set_equilibrium(self.reader.equilibrium) + self.save_phantom_profiles() @abstractmethod def setup_models(self, diagnostics: list): diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 02b20ec1..9df0efa3 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -105,51 +105,23 @@ class BayesWorkflowExample(AbstractBayesWorkflow): def __init__( - self, - pulse: int = None, - pulse_to_write: bool = None, - run: str = "RUN01", - diagnostics: list = None, - param_names: list = None, - opt_quantity: list = None, - priors: dict = None, - profile_params: dict = None, - phantoms: bool = False, - model_kwargs: dict = {}, - nwalkers: int = 50, - tstart: float = 0.02, - tend: float = 0.10, - dt: float = 0.01, - tsample: float = 0.06, - iterations: int = 100, - burn_frac: float = 0, - mds_write: bool = False, - plot: bool = True, - sample_high_density: bool = False, + self, + pulse: int = None, + diagnostics: list = None, + param_names: list = None, + opt_quantity: list = None, + priors: dict = None, + profile_params: dict = None, + phantoms: bool = False, ): self.pulse = pulse - self.pulse_to_write = pulse_to_write - self.run = run self.diagnostics = diagnostics self.param_names = param_names self.opt_quantity = opt_quantity self.priors = priors self.profile_params = profile_params - self.model_kwargs = model_kwargs self.phantoms = phantoms - self.tstart = tstart - self.tend = tend - self.dt = dt - self.tsample = tsample - self.nwalkers = nwalkers - self.iterations = iterations - self.burn_frac = burn_frac - - self.mds_write = mds_write - self.plot = plot - self.sample_high_density = sample_high_density - for attribute in [ "param_names", "opt_quantity", @@ -165,6 +137,7 @@ def __init__( "Set phantoms to True when running phantom plasma i.e. pulse=None" ) + # TODO: Add some abstraction here if pulse is None: print("Running in test mode") self.read_test_data( @@ -176,17 +149,19 @@ def __init__( ) else: self.read_data(self.diagnostics) - self.setup_plasma() - self.save_phantom_profiles() self.setup_models(self.diagnostics) - self.setup_opt_data(phantoms=self.phantoms) - self.setup_optimiser(self.model_kwargs) - def setup_plasma(self): + def setup_plasma(self, + tstart=0.02, + tend=0.10, + dt=0.005, + tsample=0.050, + ): + # TODO: move to plasma.py self.plasma = Plasma( - tstart=self.tstart, - tend=self.tend, - dt=self.dt, + tstart=tstart, + tend=tend, + dt=dt, main_ion="h", impurities=("ar", "c"), impurity_concentration=( @@ -197,11 +172,12 @@ def setup_plasma(self): n_rad=20, ) self.plasma.time_to_calculate = self.plasma.t[ - np.abs(self.tsample - self.plasma.t).argmin() + np.abs(tsample - self.plasma.t).argmin() ] self.plasma.set_equilibrium(self.equilibrium) self.plasma.update_profiles(self.profile_params) self.plasma.build_atomic_data(calc_power_loss=False) + self.save_phantom_profiles() def setup_models(self, diagnostics: list): self.models = {} @@ -230,17 +206,17 @@ def setup_models(self, diagnostics: list): elif diag == "xrcs": los_transform = self.transforms[diag] los_transform.set_equilibrium(self.plasma.equilibrium) - window_vector = None + window = None if hasattr(self, "data"): if diag in self.data.keys(): - window_vector = ( - self.data[diag]["spectra"].wavelength.values * 0.1 + window = ( + self.data[diag]["spectra"].wavelength.values * 0.1 ) model = HelikeSpectrometer( name="xrcs", window_masks=[slice(0.394, 0.396)], - window_vector=window_vector, + window=window, ) model.set_los_transform(los_transform) @@ -254,7 +230,7 @@ def setup_models(self, diagnostics: list): model.set_transect_transform(transform) else: raise ValueError(f"{diag} not found in setup_models") - model.plasma = self.plasma + # model.plasma = self.plasma TODO: add this to the models somewhere self.models[diag] = model def setup_opt_data(self, phantoms=False): @@ -268,28 +244,28 @@ def _phantom_data(self, noise=False, noise_factor=0.1): if "smmh1" in self.diagnostics: opt_data["smmh1.ne"] = ( self.models["smmh1"]() - .pop("ne") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("ne") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if "xrcs" in self.diagnostics: opt_data["xrcs.spectra"] = ( self.models["xrcs"]() - .pop("spectra") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("spectra") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) opt_data["xrcs.spectra"]["error"] = np.sqrt(opt_data["xrcs.spectra"]) if "cxff_pi" in self.diagnostics: cxrs_data = ( self.models["cxff_pi"]() - .pop("ti") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("ti") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2, drop=True) if "efit" in self.diagnostics: opt_data["efit.wp"] = ( self.models["efit"]() - .pop("wp") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("wp") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if noise: @@ -315,7 +291,7 @@ def _exp_data(self): opt_data = flatdict.FlatDict(self.data, ".") if "xrcs" in self.diagnostics: opt_data["xrcs.spectra"]["wavelength"] = ( - opt_data["xrcs.spectra"].wavelength * 0.1 + opt_data["xrcs.spectra"].wavelength * 0.1 ) background = opt_data["xrcs.spectra"].where( (opt_data["xrcs.spectra"].wavelength < 0.392) @@ -335,7 +311,19 @@ def _exp_data(self): ) return opt_data - def setup_optimiser(self, model_kwargs): + def setup_optimiser(self, + model_kwargs, + iterations=200, + nwalkers=50, + burn_frac=0.10, + sample_high_density=False, + ): + self.model_kwargs = model_kwargs + self.iterations = iterations + self.nwalkers = nwalkers + self.burn_frac = burn_frac + self.sample_high_density = sample_high_density + self.bayesmodel = BayesModels( plasma=self.plasma, data=self.opt_data, @@ -369,11 +357,13 @@ def _sample_start_points(self, sample_high_density: bool = True): ) return start_points - def __call__(self, filepath="./results/test/", **kwargs): - if self.mds_write: + def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_to_write=None, plot=False, + **kwargs): + if mds_write: # check_analysis_run(self.pulse, self.run) self.node_structure = create_nodes( - pulse_to_write=self.pulse_to_write, + pulse_to_write=pulse_to_write, + run=run, diagnostic_quantities=self.opt_quantity, mode="NEW", ) @@ -381,11 +371,11 @@ def __call__(self, filepath="./results/test/", **kwargs): self.run_sampler() self.save_pickle(filepath=filepath) - if self.plot: # currently requires result with DataArrays + if plot: # currently requires result with DataArrays plot_bayes_result(self.result, filepath) self.result = self.dict_of_dataarray_to_numpy(self.result) - if self.mds_write: + if mds_write: write_nodes(self.pulse_to_write, self.node_structure, self.result) return self.result @@ -394,26 +384,34 @@ def __call__(self, filepath="./results/test/", **kwargs): if __name__ == "__main__": run = BayesWorkflowExample( pulse=None, - pulse_to_write=23000101, - run="RUN01", + phantoms=True, diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], opt_quantity=OPTIMISED_QUANTITY, param_names=OPTIMISED_PARAMS, profile_params=DEFAULT_PROFILE_PARAMS, priors=DEFAULT_PRIORS, - model_kwargs={ - "xrcs_moment_analysis": False, - }, - phantoms=True, + ) + + run.setup_plasma( + tstart=0.02, + tend=0.10, + dt=0.005, + tsample=0.060, + ) + run.setup_opt_data(phantoms=run.phantoms) + run.setup_optimiser( iterations=200, nwalkers=50, burn_frac=0.10, - dt=0.005, - tsample=0.060, - mds_write=True, - plot=True, sample_high_density=False, + model_kwargs={ + "xrcs_moment_analysis": False, + } ) results = run( filepath="./results/test/", + pulse_to_write=23000101, + run="RUN01", + mds_write=True, + plot=True, ) From 3f5c089626aa66d3bccbc3edfbc4c17434b4c9dc Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 11 Aug 2023 14:00:37 +0100 Subject: [PATCH 124/288] stashing --- indica/workflows/bayes_workflow_example.py | 5 +- .../workflows/test_bayes_workflow_example.py | 51 ++++++++++++------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 9df0efa3..79c34df7 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -230,7 +230,7 @@ def setup_models(self, diagnostics: list): model.set_transect_transform(transform) else: raise ValueError(f"{diag} not found in setup_models") - # model.plasma = self.plasma TODO: add this to the models somewhere + model.plasma = self.plasma #TODO: add this to the models somewhere self.models[diag] = model def setup_opt_data(self, phantoms=False): @@ -390,6 +390,9 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ param_names=OPTIMISED_PARAMS, profile_params=DEFAULT_PROFILE_PARAMS, priors=DEFAULT_PRIORS, + tstart=0.02, + tend=0.10, + dt=0.005, ) run.setup_plasma( diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index 2da073e9..5861c49a 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -21,30 +21,41 @@ class TestBayesWorkflowExample: def setup_class(self): - self.default_settings = dict( + self.init_settings = dict( pulse=None, - pulse_to_write=23000111, - run="RUN01", + phantoms = True, diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], opt_quantity=OPTIMISED_QUANTITY, param_names=OPTIMISED_PARAMS, profile_params=DEFAULT_PROFILE_PARAMS, priors=DEFAULT_PRIORS, - model_kwargs={ - "xrcs_moment_analysis": False, - }, - phantoms=True, - iterations=1, - nwalkers=20, - burn_frac=0.10, + tstart=0.02, + tend=0.10, dt=0.005, - tsample=0.060, - mds_write=False, - plot=False, - sample_high_density=False, ) + self.plasma_settings = dict( + + tsample=0.060,) + + + self.opt_settings = dict( + model_kwargs = { + "xrcs_moment_analysis": False, + }, + iterations = 1, + nwalkers = 20, + burn_frac = 0.10, + sample_high_density = False, + ) + + self.call_settings = dict( + filepath=None, + pulse_to_write=23000101, + run="RUN01", + mds_write = False, + plot = False,) - self.untouched_workflow = BayesWorkflowExample(**self.default_settings) + self.untouched_workflow = BayesWorkflowExample(**self.init_settings) self.workflow = copy.deepcopy(self.untouched_workflow) def setup_method(self): @@ -55,13 +66,10 @@ def teardown_method(self): def test_workflow_initializes(self): attributes_to_check = [ - "plasma", - "phantom_profiles", "data", "reader", - "opt_data", "models", - "bayesmodel", + "equilibrium" ] for attribute in attributes_to_check: if not hasattr(self.workflow, attribute): @@ -84,9 +92,11 @@ def test_init_not_including_all_required_inputs(self): # assert all(diag_name in self.workflow.reader.keys() for diag_name in self.workflow.diagnostics) def test_plasma_has_equilibrium(self): + self.workflow.setup_plasma() assert hasattr(self.workflow.plasma, "equilibrium") def test_phantom_profiles_are_not_mutatable(self): + self.workflow.setup_plasma() phantoms = copy.deepcopy(self.workflow.phantom_profiles) self.workflow.plasma.electron_temperature += 1 assert phantoms is not self.workflow.phantom_profiles @@ -126,6 +136,9 @@ def test_sampling_from_priors(self): assert True def test_worklow_has_results_after_run(self): + self.workflow.setup_plasma() + self.workflow.setup_opt_data(self.workflow.phantoms) + self.workflow.setup_optimiser() self.workflow.run_sampler() if not hasattr(self.workflow, "result"): raise ValueError(f"missing result in workflow object") From 766825a8e5ce315fc3db4742272ce5551f2ed2ee Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 15 Aug 2023 10:58:02 +0100 Subject: [PATCH 125/288] setup_plasma now takes kwargs --- indica/workflows/bayes_workflow_example.py | 62 ++++++++++++++-------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 79c34df7..0ce4f073 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -113,6 +113,9 @@ def __init__( priors: dict = None, profile_params: dict = None, phantoms: bool = False, + tstart=0.02, + tend=0.10, + dt=0.005, ): self.pulse = pulse self.diagnostics = diagnostics @@ -121,6 +124,9 @@ def __init__( self.priors = priors self.profile_params = profile_params self.phantoms = phantoms + self.tstart = tstart + self.tend = tend + self.dt = dt for attribute in [ "param_names", @@ -152,25 +158,34 @@ def __init__( self.setup_models(self.diagnostics) def setup_plasma(self, - tstart=0.02, - tend=0.10, - dt=0.005, + tstart=None, + tend=None, + dt=None, tsample=0.050, + main_ion = "h", + impurities = ("ar", "c"), + impurity_concentration = (0.001, 0.04), + n_rad=20, + **kwargs ): + + if not all([tstart, tend, dt]): + tstart = self.tstart + tend = self.tend + dt = self.dt + # TODO: move to plasma.py self.plasma = Plasma( tstart=tstart, tend=tend, dt=dt, - main_ion="h", - impurities=("ar", "c"), - impurity_concentration=( - 0.001, - 0.04, - ), + main_ion=main_ion, + impurities=impurities, + impurity_concentration=impurity_concentration, full_run=False, - n_rad=20, + n_rad=n_rad, ) + self.tsample = tsample self.plasma.time_to_calculate = self.plasma.t[ np.abs(tsample - self.plasma.t).argmin() ] @@ -199,13 +214,13 @@ def setup_models(self, diagnostics: list): # machine_dimensions=machine_dims, # passes=2, # ) - los_transform.set_equilibrium(self.plasma.equilibrium) + los_transform.set_equilibrium(self.equilibrium) model = Interferometry(name=diag) model.set_los_transform(los_transform) elif diag == "xrcs": los_transform = self.transforms[diag] - los_transform.set_equilibrium(self.plasma.equilibrium) + los_transform.set_equilibrium(self.equilibrium) window = None if hasattr(self, "data"): if diag in self.data.keys(): @@ -225,21 +240,25 @@ def setup_models(self, diagnostics: list): elif diag == "cxff_pi": transform = self.transforms[diag] - transform.set_equilibrium(self.plasma.equilibrium) + transform.set_equilibrium(self.equilibrium) model = ChargeExchange(name=diag, element="ar") model.set_transect_transform(transform) else: raise ValueError(f"{diag} not found in setup_models") - model.plasma = self.plasma #TODO: add this to the models somewhere self.models[diag] = model - def setup_opt_data(self, phantoms=False): + def setup_opt_data(self, phantoms=False, **kwargs): + if not hasattr(self, "plasma"): + raise ValueError("Missing plasma object required for setup_opt_data") + for model in self.models.values(): # Maybe refactor here... + model.plasma = self.plasma + if phantoms: - self.opt_data = self._phantom_data() + self.opt_data = self._phantom_data(**kwargs) else: - self.opt_data = self._exp_data() + self.opt_data = self._exp_data(**kwargs) - def _phantom_data(self, noise=False, noise_factor=0.1): + def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): opt_data = {} if "smmh1" in self.diagnostics: opt_data["smmh1.ne"] = ( @@ -269,7 +288,7 @@ def _phantom_data(self, noise=False, noise_factor=0.1): ) if noise: - opt_data["smmh1.ne"] = opt_data["smmh1.ne"] + self.opt_data[ + opt_data["smmh1.ne"] = opt_data["smmh1.ne"] + opt_data[ "smmh1.ne" ].max().values * np.random.normal(0, noise_factor, None) opt_data["xrcs.spectra"] = opt_data["xrcs.spectra"] + np.random.normal( @@ -287,7 +306,7 @@ def _phantom_data(self, noise=False, noise_factor=0.1): ].max().values * np.random.normal(0, noise_factor, None) return opt_data - def _exp_data(self): + def _exp_data(self, **kwargs): opt_data = flatdict.FlatDict(self.data, ".") if "xrcs" in self.diagnostics: opt_data["xrcs.spectra"]["wavelength"] = ( @@ -396,9 +415,6 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ ) run.setup_plasma( - tstart=0.02, - tend=0.10, - dt=0.005, tsample=0.060, ) run.setup_opt_data(phantoms=run.phantoms) From cb93ae8c724cb14349de25aaa9734e7df5ec8cdc Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 15 Aug 2023 10:59:05 +0100 Subject: [PATCH 126/288] adding plasma to models now happens in setup_opt_data --- indica/workflows/abstract_bayes_workflow.py | 135 ++++++++++---------- 1 file changed, 69 insertions(+), 66 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index d9ff7184..b40a9db8 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -12,13 +12,13 @@ class AbstractBayesWorkflow(ABC): @abstractmethod def __init__( - self, - pulse=None, - phantoms=None, - diagnostics=None, - param_names=None, - opt_quantity=None, - priors=None, + self, + pulse=None, + phantoms=None, + diagnostics=None, + param_names=None, + opt_quantity=None, + priors=None, ): self.pulse = pulse self.phantoms = phantoms @@ -81,6 +81,9 @@ def setup_opt_data(self, phantom: bool = False): """ Get and prepare the data in necessary format for optimiser """ + for model in self.models: + model.plasma = self.plasma + if phantom: self.opt_data = self._phantom_data() else: @@ -107,8 +110,8 @@ def save_phantom_profiles(self, kinetic_profiles=None): if self.phantoms: phantom_profiles = { profile_key: getattr(self.plasma, profile_key) - .sel(t=self.plasma.time_to_calculate) - .copy() + .sel(t=self.plasma.time_to_calculate) + .copy() for profile_key in kinetic_profiles } else: @@ -116,7 +119,7 @@ def save_phantom_profiles(self, kinetic_profiles=None): profile_key: getattr(self.plasma, profile_key).sel( t=self.plasma.time_to_calculate ) - * 0 + * 0 for profile_key in kinetic_profiles } @@ -201,7 +204,7 @@ def _build_result_dict(self): } result["PHANTOMS"].update( { - f"NIZ{num_imp+1}": self.phantom_profiles["impurity_density"].sel( + f"NIZ{num_imp + 1}": self.phantom_profiles["impurity_density"].sel( element=imp ) for num_imp, imp in enumerate(self.plasma.impurities) @@ -221,40 +224,40 @@ def _build_result_dict(self): "RHO_TOR": self.plasma.equilibrium.rhotor.interp(t=self.plasma.t), "NE": self.blobs["electron_density"].median(dim="index"), "NI": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .median(dim="index"), "TE": self.blobs["electron_temperature"].median(dim="index"), "TI": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .median(dim="index"), "NFAST": self.blobs["fast_density"].median(dim="index"), "NNEUTR": self.blobs["neutral_density"].median(dim="index"), "NE_ERR": self.blobs["electron_density"].std(dim="index"), "NI_ERR": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .std(dim="index"), "TE_ERR": self.blobs["electron_temperature"].std(dim="index"), "TI_ERR": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .std(dim="index"), "NFAST_ERR": self.blobs["fast_density"].std(dim="index"), "NNEUTR_ERR": self.blobs["neutral_density"].std(dim="index"), } result["PROFILES"] = { **result["PROFILES"], **{ - f"NIZ{num_imp+1}": self.blobs["impurity_density"] - .sel(element=imp) - .median(dim="index") + f"NIZ{num_imp + 1}": self.blobs["impurity_density"] + .sel(element=imp) + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } result["PROFILES"] = { **result["PROFILES"], **{ - f"NIZ{num_imp+1}_ERR": self.blobs["impurity_density"] - .sel(element=imp) - .std(dim="index") + f"NIZ{num_imp + 1}_ERR": self.blobs["impurity_density"] + .sel(element=imp) + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -262,8 +265,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"TIZ{num_imp + 1}": self.blobs["ion_temperature"] - .sel(element=imp) - .median(dim="index") + .sel(element=imp) + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -271,8 +274,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"TIZ{num_imp + 1}_ERR": self.blobs["ion_temperature"] - .sel(element=imp) - .std(dim="index") + .sel(element=imp) + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -290,7 +293,7 @@ def _build_result_dict(self): result["PROFILE_STAT"] = { **result["PROFILE_STAT"], **{ - f"NIZ{num_imp+1}": self.blobs["impurity_density"].sel(element=imp) + f"NIZ{num_imp + 1}": self.blobs["impurity_density"].sel(element=imp) for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -311,41 +314,41 @@ def _build_result_dict(self): result["GLOBAL"] = { "TI0": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "TE0": self.blobs["electron_temperature"] - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "NE0": self.blobs["electron_density"] - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "NI0": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "TI0_ERR": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), "TE0_ERR": self.blobs["electron_temperature"] - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), "NE0_ERR": self.blobs["electron_density"] - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), "NI0_ERR": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), } result["GLOBAL"] = { **result["GLOBAL"], **{ f"TI0Z{num_imp + 1}": self.blobs["ion_temperature"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -353,9 +356,9 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"TI0Z{num_imp + 1}_ERR": self.blobs["ion_temperature"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -363,9 +366,9 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"NI0Z{num_imp + 1}": self.blobs["impurity_density"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -373,9 +376,9 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"NI0Z{num_imp + 1}_ERR": self.blobs["impurity_density"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -452,9 +455,9 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl autocorr = np.ones(shape=(iterations, n_params)) * np.nan old_tau = np.inf for sample in sampler.sample( - start_points, - iterations=iterations, - progress=True, + start_points, + iterations=iterations, + progress=True, ): if sampler.iteration % auto_sample: continue @@ -465,5 +468,5 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl if converged: break old_tau = new_tau - autocorr = autocorr[: sampler.iteration,] + autocorr = autocorr[: sampler.iteration, ] return autocorr From cf1c127c6128b38200011131be7a56df09d82841 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 15 Aug 2023 10:59:28 +0100 Subject: [PATCH 127/288] workflow broken up into methods --- .../workflows/test_bayes_workflow_example.py | 66 +++++++++++-------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index 5861c49a..b80fc392 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -12,10 +12,7 @@ """ TODO: - -test kwarg handling - -Mock reader for testing experimental data methods +Mock reader for testing experimental data reading """ @@ -34,18 +31,18 @@ def setup_class(self): dt=0.005, ) self.plasma_settings = dict( - - tsample=0.060,) + tsample=0.060, + ) - self.opt_settings = dict( + self.optimiser_settings = dict( model_kwargs = { "xrcs_moment_analysis": False, }, - iterations = 1, - nwalkers = 20, - burn_frac = 0.10, - sample_high_density = False, + iterations = 1, + nwalkers = 20, + burn_frac = 0.10, + sample_high_density = False, ) self.call_settings = dict( @@ -55,11 +52,11 @@ def setup_class(self): mds_write = False, plot = False,) - self.untouched_workflow = BayesWorkflowExample(**self.init_settings) - self.workflow = copy.deepcopy(self.untouched_workflow) + self.workflow_untouched = BayesWorkflowExample(**self.init_settings) + self.workflow=None def setup_method(self): - self.workflow = copy.deepcopy(self.untouched_workflow) + self.workflow = copy.deepcopy(self.workflow_untouched) def teardown_method(self): self.workflow = None @@ -79,34 +76,39 @@ def test_workflow_initializes(self): def test_init_phantoms_false_with_example_plasma(self): with pytest.raises(ValueError): example = BayesWorkflowExample( - dict(self.default_settings, **{"phantoms": False}) + dict(self.init_settings, **{"phantoms": False}) ) def test_init_not_including_all_required_inputs(self): with pytest.raises(ValueError): example = BayesWorkflowExample( - dict(self.default_settings, **{"param_names": None}) + dict(self.init_settings, **{"param_names": None}) ) # def test_reader_has_read_all_diagnostic_data(self): # assert all(diag_name in self.workflow.reader.keys() for diag_name in self.workflow.diagnostics) def test_plasma_has_equilibrium(self): - self.workflow.setup_plasma() + self.workflow.setup_plasma(**self.plasma_settings) assert hasattr(self.workflow.plasma, "equilibrium") def test_phantom_profiles_are_not_mutatable(self): - self.workflow.setup_plasma() + self.workflow.setup_plasma(**self.plasma_settings) phantoms = copy.deepcopy(self.workflow.phantom_profiles) self.workflow.plasma.electron_temperature += 1 assert phantoms is not self.workflow.phantom_profiles def test_setup_models_with_wrong_diagnostic_names(self): - with pytest.raises(Exception): + with pytest.raises(ValueError): self.workflow.setup_models(["foo", "bar", "xrcs"]) + def test_opt_data_without_plasma(self): + with pytest.raises(ValueError): + self.workflow.setup_opt_data(phantoms=True) + def test_phantom_data_exists(self): - self.workflow._phantom_data() + self.workflow.setup_plasma(**self.plasma_settings) + self.workflow.setup_opt_data(phantoms=True) assert self.workflow.opt_data # def test_experimental_data_exists(self): @@ -114,7 +116,8 @@ def test_phantom_data_exists(self): # assert self.workflow.opt_data def test_phantom_data_has_time_dim(self): - self.workflow._phantom_data() + self.workflow.setup_plasma(**self.plasma_settings) + self.workflow.setup_opt_data(phantoms=True) for key, value in self.workflow.opt_data.items(): assert "t" in value.dims @@ -124,21 +127,26 @@ def test_phantom_data_has_time_dim(self): # assert "t" in value.dims def test_phantom_data_runs_with_noise_added(self): - self.workflow._phantom_data(noise=True) + self.workflow.setup_plasma(**self.plasma_settings) + self.workflow.setup_opt_data(phantoms=True, noise=True) assert self.workflow.opt_data - def test_sampling_from_high_density(self): - self.workflow._sample_start_points(sample_high_density=True) + def test_sampling_from_priors(self): + self.workflow.setup_plasma(**self.plasma_settings) + self.workflow.setup_opt_data(phantoms=True, ) + self.workflow.setup_optimiser(**dict(self.optimiser_settings, **{"sample_high_density":False})) assert True - def test_sampling_from_priors(self): - self.workflow._sample_start_points(sample_high_density=False) + def test_sampling_from_high_density(self): + self.workflow.setup_plasma(**self.plasma_settings) + self.workflow.setup_opt_data(phantoms=True, ) + self.workflow.setup_optimiser(**dict(self.optimiser_settings, **{"sample_high_density":True})) assert True def test_worklow_has_results_after_run(self): - self.workflow.setup_plasma() - self.workflow.setup_opt_data(self.workflow.phantoms) - self.workflow.setup_optimiser() + self.workflow.setup_plasma(**self.plasma_settings) + self.workflow.setup_opt_data(phantoms=True) + self.workflow.setup_optimiser(**self.optimiser_settings) self.workflow.run_sampler() if not hasattr(self.workflow, "result"): raise ValueError(f"missing result in workflow object") From 260a5150812f4345246add54d20b217b332d4e58 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 15 Aug 2023 14:08:48 +0100 Subject: [PATCH 128/288] removed redundant kwargs --- indica/models/helike_spectroscopy.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index b2dcb6fc..bfb7c406 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -4,7 +4,6 @@ import xarray as xr from xarray import DataArray -from indica.physics import ev_doppler from indica.converters.line_of_sight import LineOfSightTransform from indica.datatypes import ELEMENTS from indica.models.abstractdiagnostic import DiagnosticModel @@ -38,8 +37,6 @@ def __init__( instrument_method="get_helike_spectroscopy", etendue: float = 1.0, calibration: float = 8.0e-20, - marchuk: bool = True, - full_run: bool = False, element: str = "ar", window_len: int = 1030, window_lim: list = [0.394, 0.401], @@ -70,7 +67,8 @@ def __init__( self.line_ranges = LINE_RANGES self.line_labels = line_labels - window = np.linspace(window_lim[0], window_lim[1], window_len) + if window is None: + window = np.linspace(window_lim[0], window_lim[1], window_len) mask = np.zeros(shape=window.shape) if self.window_masks: for mslice in self.window_masks: From 0821b9dc62b70471493ae5cfa368e99dfb147b86 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 15 Aug 2023 14:40:50 +0100 Subject: [PATCH 129/288] fixed non_plasma call option --- indica/models/helike_spectroscopy.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index bfb7c406..87ce54d2 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -39,10 +39,10 @@ def __init__( calibration: float = 8.0e-20, element: str = "ar", window_len: int = 1030, - window_lim: list = [0.394, 0.401], + window_lim=None, window: np.array = None, - window_masks: list = [], - line_labels: list = ["w", "k", "n3", "n345", "z", "qra"], + window_masks=None, + line_labels=None, ): """ Read all atomic data and initialise objects @@ -53,14 +53,19 @@ def __init__( String identifier for the spectrometer """ + if window_lim is None: + window_lim = [0.394, 0.401] + if window_masks is None: + window_masks = [] + if line_labels is None: + line_labels = ["w", "k", "n3", "n345", "z", "qra"] + self.name = name self.instrument_method = instrument_method - self.element: str = element z_elem, a_elem, name_elem = ELEMENTS[element] self.ion_charge: int = z_elem - 2 # He-like self.ion_mass: float = a_elem - self.etendue = etendue self.calibration = calibration self.window_masks = window_masks @@ -408,7 +413,7 @@ def __call__( "moment_analysis cannot be used when window_masks is not set to None" ) - if self.plasma is not None: + if hasattr(self, "plasma"): if t is None: t = self.plasma.time_to_calculate Te = self.plasma.electron_temperature.interp( From 78800c6704b0219fe427de419e89240bdf18d9ac Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 15 Aug 2023 15:22:00 +0100 Subject: [PATCH 130/288] fixed plasma_initialisation --- tests/unit/models/test_plasma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/models/test_plasma.py b/tests/unit/models/test_plasma.py index 9b2a8fb7..4746e702 100644 --- a/tests/unit/models/test_plasma.py +++ b/tests/unit/models/test_plasma.py @@ -35,7 +35,7 @@ def setup_class(self): # return def teardown_method(self): - self.plasma.initialize_variables(tstart=self.tstart, tend=self.tend, dt=self.dt) + self.plasma.initialize_variables() def test_plasma_initializes(self): assert hasattr(self, "plasma") From 62d67277d4e6183c6e5ac407b8ae1b1374974d3c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 15 Aug 2023 16:24:29 +0100 Subject: [PATCH 131/288] fixing violin plotting --- indica/workflows/bayes_plots.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 8695969d..c027c004 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -172,16 +172,18 @@ def violinplot( # quantiles=[0.025, 0.975, 0.16, 0.84], # showmedians=True, ) + y = diag_data[key].sel(t=data[key].t).values + # TODO abstract the yerr + axs.errorbar( + 1, y=y, yerr=y * 0.10, fmt="D", ecolor="black", capsize=10, color="black" + ) violin["bodies"][0].set_edgecolor("black") axs.set_xlabel(key) top = axs.get_ylim()[1] bot = axs.get_ylim()[0] axs.set_ylim(top=top * 1.1, bottom=bot * 0.9) axs.set_ylabel(f"{ylabel}") - y = diag_data[key].sel(t=data[key].t).values - axs.errorbar( - 1, y=y, yerr=y * 0.10, fmt="D", ecolor="black", capsize=10, color="black" - ) + set_axis_sci() plt.setp([axs.get_xticklabels()], visible=False) plt.savefig(figheader + filename) From 98190dda0476bb01c175cefc6ed56e1ce6e9a935 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 15 Aug 2023 16:44:34 +0100 Subject: [PATCH 132/288] units for xrcs.spectra.wavelength fixed --- indica/workflows/bayes_workflow_example.py | 48 +++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 0ce4f073..e2e12be5 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -7,6 +7,7 @@ from indica.workflows.bayes_plots import plot_bayes_result from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow from indica.writers.bda_tree import create_nodes, write_nodes, check_analysis_run +from indica.converters.line_of_sight import LineOfSightTransform from indica.models.interferometry import Interferometry, smmh1_transform_example from indica.models.helike_spectroscopy import ( @@ -97,7 +98,7 @@ OPTIMISED_QUANTITY = [ "xrcs.spectra", - "cxff_pi.ti", + # "cxff_pi.ti", "efit.wp", "smmh1.ne", ] @@ -198,22 +199,22 @@ def setup_models(self, diagnostics: list): self.models = {} for diag in diagnostics: if diag == "smmh1": - los_transform = self.transforms[diag] - # machine_dims = self.plasma.machine_dimensions - # origin = np.array([[-0.38063365, 0.91893092, 0.01]]) - # # end = np.array([[0, 0, 0.01]]) - # direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) - # los_transform = LineOfSightTransform( - # origin[:, 0], - # origin[:, 1], - # origin[:, 2], - # direction[:, 0], - # direction[:, 1], - # direction[:, 2], - # name="", - # machine_dimensions=machine_dims, - # passes=2, - # ) + # los_transform = self.transforms[diag] + machine_dims = ((0.15, 0.95), (-0.7, 0.7)) + origin = np.array([[-0.38063365, 0.91893092, 0.01]]) + # end = np.array([[0, 0, 0.01]]) + direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) + los_transform = LineOfSightTransform( + origin[:, 0], + origin[:, 1], + origin[:, 2], + direction[:, 0], + direction[:, 1], + direction[:, 2], + name="", + machine_dimensions=machine_dims, + passes=2, + ) los_transform.set_equilibrium(self.equilibrium) model = Interferometry(name=diag) model.set_los_transform(los_transform) @@ -225,7 +226,7 @@ def setup_models(self, diagnostics: list): if hasattr(self, "data"): if diag in self.data.keys(): window = ( - self.data[diag]["spectra"].wavelength.values * 0.1 + self.data[diag]["spectra"].wavelength.values ) model = HelikeSpectrometer( @@ -395,16 +396,16 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ self.result = self.dict_of_dataarray_to_numpy(self.result) if mds_write: - write_nodes(self.pulse_to_write, self.node_structure, self.result) + write_nodes(pulse_to_write, self.node_structure, self.result) return self.result if __name__ == "__main__": run = BayesWorkflowExample( - pulse=None, + pulse=9780, phantoms=True, - diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], + diagnostics=["xrcs", "efit", "smmh1",], opt_quantity=OPTIMISED_QUANTITY, param_names=OPTIMISED_PARAMS, profile_params=DEFAULT_PROFILE_PARAMS, @@ -419,12 +420,11 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ ) run.setup_opt_data(phantoms=run.phantoms) run.setup_optimiser( - iterations=200, + iterations=2, nwalkers=50, burn_frac=0.10, - sample_high_density=False, + sample_high_density=True, model_kwargs={ - "xrcs_moment_analysis": False, } ) results = run( From f732da034d92817e50b9e5e239501d305113e355 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 09:25:10 +0100 Subject: [PATCH 133/288] nchannels added to example_los --- indica/models/interferometry.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/indica/models/interferometry.py b/indica/models/interferometry.py index 2f6393e0..c9354205 100644 --- a/indica/models/interferometry.py +++ b/indica/models/interferometry.py @@ -89,9 +89,12 @@ def __call__( return self.bckc -def smmh1_transform_example(): - los_start = np.array([[0.8, 0, 0], [0.8, 0, -0.1], [0.8, 0, -0.2]]) - los_end = np.array([[0.17, 0, 0], [0.17, 0, -0.25], [0.17, 0, -0.2]]) +def smmh1_transform_example(nchannels): + + los_start = np.array([[0.8, 0, 0]]) * np.ones((nchannels,3)) + los_start[:,2] = np.linspace(0, -0.2, nchannels) + los_end = np.array([[0.17, 0, 0]]) * np.ones((nchannels, 3)) + los_end[:, 2] = np.linspace(0, -0.2, nchannels) origin = los_start direction = los_end - los_start los_transform = LineOfSightTransform( @@ -117,7 +120,7 @@ def example_run(pulse: int = None, plasma=None, plot=False): model = Interferometry( diagnostic_name, ) - los_transform = smmh1_transform_example() + los_transform = smmh1_transform_example(3) los_transform.set_equilibrium(plasma.equilibrium) model.set_los_transform(los_transform) model.set_plasma(plasma) From ec1c9cadd5d1b2231772239142d4b139c8267b3e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 09:26:41 +0100 Subject: [PATCH 134/288] adding print message for fake data reading --- indica/workflows/abstract_bayes_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index b40a9db8..71b886ad 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -31,6 +31,7 @@ def __init__( def read_test_data(self, diagnostic_transforms: dict): # Used with phantom data for purposes of tests + print("Reading fake data") self.reader = None self.equilibrium = fake_equilibrium( self.tstart, From b0fac373937554d9938a5f0285f5cb7ec907fcac Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 09:59:50 +0100 Subject: [PATCH 135/288] adjusting example los --- indica/models/helike_spectroscopy.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 87ce54d2..5fd32aae 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -465,12 +465,13 @@ def __call__( self._build_bckc_dictionary() return self.bckc -def helike_transform_example(nchannels=3): +def helike_transform_example(nchannels): los_end = np.full((nchannels, 3), 0.0) los_end[:, 0] = 0.17 los_end[:, 1] = 0.0 - los_end[:, 2] = np.linspace(0.43, -0.43, nchannels) - los_start = np.array([[0.8, 0, 0]] * los_end.shape[0]) + los_end[:, 2] = np.linspace(0.2, -0.5, nchannels) + los_start = np.array([[0.9, 0, 0]] * los_end.shape[0]) + los_start[:, 2] = -0.1 origin = los_start direction = los_end - los_start @@ -531,8 +532,7 @@ def example_run( plt.ylabel("spectra") plt.legend() - # model.los_transform.plot(tplot, - # plot_all=model.los_transform.x1.__len__() > 1) + los_transform.plot() if "int_w" in bckc.keys() & "t" in bckc["int_w"].dims: plt.figure() From 03d68c70efc37bc1a50e1b981827dfd6b08a7ebd Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 10:12:45 +0100 Subject: [PATCH 136/288] bckc method not printing spectra/fit not available everytime --- indica/models/charge_exchange.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/indica/models/charge_exchange.py b/indica/models/charge_exchange.py index 9d223131..5551d8a6 100644 --- a/indica/models/charge_exchange.py +++ b/indica/models/charge_exchange.py @@ -42,6 +42,12 @@ def _build_bckc_dictionary(self): self.bckc[quantity] = self.Ti_at_channels long_name = "Ion temperature" units = "eV" + elif quant == "spectra": + # Placeholder + continue + elif quant == "fit": + # Placeholder + continue else: print(f"{quant} not available in model for {self.instrument_method}") continue From 72e6e7eb306a2545abedd634e290b2472aa7c8b5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 10:41:38 +0100 Subject: [PATCH 137/288] adjusting priors --- indica/workflows/bayes_workflow_example.py | 84 +++++++++++----------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index e2e12be5..5d2e5868 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -26,8 +26,8 @@ "Ne_prof.y1": 2e18, "Ne_prof.yend": 1e18, "Ne_prof.wped": 2, - "Nimp_prof.y0": 3e16, - "Nimp_prof.y1": 0.5e16, + "Nimp_prof.y0": 1e17, + "Nimp_prof.y1": 5e15, "Nimp_prof.wcenter": 0.4, "Nimp_prof.wped": 6, "Nimp_prof.peaking": 2, @@ -50,28 +50,30 @@ "Ne_prof.wped": get_uniform(1, 6), "Ne_prof.wcenter": get_uniform(0.1, 0.8), "Ne_prof.peaking": get_uniform(1, 6), - "ar_conc": loguniform(0.0001, 0.01), + "Nimp_prof.y0": loguniform(1e16, 1e18), - "Nimp_prof.y1": get_uniform(1e15, 2e16), + "Nimp_prof.y1": get_uniform(1e15, 1e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 ), "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), "Nimp_prof.wped": get_uniform(1, 6), "Nimp_prof.wcenter": get_uniform(0.1, 0.8), - "Nimp_prof.peaking": get_uniform(1, 20), + "Nimp_prof.peaking": get_uniform(1, 10), "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where( (x1 > x2), 1, 0 ), # impurity always more peaked + "Te_prof.y0": get_uniform(1000, 5000), "Te_prof.wped": get_uniform(1, 6), - "Te_prof.wcenter": get_uniform(0.1, 0.6), - "Te_prof.peaking": get_uniform(1, 6), - "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode - "Ti_prof.y0": get_uniform(3000, 10000), + "Te_prof.wcenter": get_uniform(0.1, 0.8), + "Te_prof.peaking": get_uniform(1, 10), + + # "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode + "Ti_prof.y0": get_uniform(1000, 10000), "Ti_prof.wped": get_uniform(1, 6), - "Ti_prof.wcenter": get_uniform(0.1, 0.6), - "Ti_prof.peaking": get_uniform(1, 20), + "Ti_prof.wcenter": get_uniform(0.1, 0.8), + "Ti_prof.peaking": get_uniform(1, 10), } OPTIMISED_PARAMS = [ @@ -80,29 +82,24 @@ # "Ne_prof.peaking", # "Ne_prof.wcenter", # "Ne_prof.wped", - # "ar_conc", + # "Nimp_prof.y1", "Nimp_prof.y0", # "Nimp_prof.wcenter", # "Nimp_prof.wped", # "Nimp_prof.peaking", + "Te_prof.y0", # "Te_prof.wped", "Te_prof.wcenter", "Te_prof.peaking", + "Ti_prof.y0", - "Ti_prof.wped", + # "Ti_prof.wped", "Ti_prof.wcenter", "Ti_prof.peaking", ] -OPTIMISED_QUANTITY = [ - "xrcs.spectra", - # "cxff_pi.ti", - "efit.wp", - "smmh1.ne", -] - class BayesWorkflowExample(AbstractBayesWorkflow): def __init__( @@ -150,7 +147,7 @@ def __init__( self.read_test_data( { "xrcs": helike_transform_example(1), - "smmh1": smmh1_transform_example(), + "smmh1": smmh1_transform_example(1), "cxff_pi": pi_transform_example(5), } ) @@ -199,22 +196,22 @@ def setup_models(self, diagnostics: list): self.models = {} for diag in diagnostics: if diag == "smmh1": - # los_transform = self.transforms[diag] - machine_dims = ((0.15, 0.95), (-0.7, 0.7)) - origin = np.array([[-0.38063365, 0.91893092, 0.01]]) - # end = np.array([[0, 0, 0.01]]) - direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) - los_transform = LineOfSightTransform( - origin[:, 0], - origin[:, 1], - origin[:, 2], - direction[:, 0], - direction[:, 1], - direction[:, 2], - name="", - machine_dimensions=machine_dims, - passes=2, - ) + los_transform = self.transforms[diag] + # machine_dims = ((0.15, 0.95), (-0.7, 0.7)) + # origin = np.array([[-0.38063365, 0.91893092, 0.01]]) + # # end = np.array([[0, 0, 0.01]]) + # direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) + # los_transform = LineOfSightTransform( + # origin[:, 0], + # origin[:, 1], + # origin[:, 2], + # direction[:, 0], + # direction[:, 1], + # direction[:, 2], + # name="", + # machine_dimensions=machine_dims, + # passes=2, + # ) los_transform.set_equilibrium(self.equilibrium) model = Interferometry(name=diag) model.set_los_transform(los_transform) @@ -403,10 +400,15 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ if __name__ == "__main__": run = BayesWorkflowExample( - pulse=9780, + pulse=None, phantoms=True, - diagnostics=["xrcs", "efit", "smmh1",], - opt_quantity=OPTIMISED_QUANTITY, + diagnostics=["xrcs", "efit", "smmh1","cxff_pi"], + opt_quantity=[ + "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne", + ], param_names=OPTIMISED_PARAMS, profile_params=DEFAULT_PROFILE_PARAMS, priors=DEFAULT_PRIORS, @@ -420,7 +422,7 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ ) run.setup_opt_data(phantoms=run.phantoms) run.setup_optimiser( - iterations=2, + iterations=100, nwalkers=50, burn_frac=0.10, sample_high_density=True, From d5ebe08c49ba9d0dad29bfd726ecee3e9aff214f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 11:19:32 +0100 Subject: [PATCH 138/288] moved kwargs to sample function --- indica/workflows/abstract_bayes_workflow.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 71b886ad..0f8afcfc 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -387,7 +387,7 @@ def _build_result_dict(self): self.result = result return self.result - def run_sampler(self): + def run_sampler(self, iterations, burn_frac): """ TODO: unsure if keeping in abstract class is best practice @@ -398,15 +398,18 @@ def run_sampler(self): result in MDSPlus node formatting """ + self.burn_frac = burn_frac + self.iterations = iterations + self.autocorr = sample_with_autocorr( self.sampler, self.start_points, - self.iterations, + iterations, self.param_names.__len__(), auto_sample=10, ) blobs = self.sampler.get_blobs( - discard=int(self.iterations * self.burn_frac), flat=True + discard=int(iterations * burn_frac), flat=True ) blob_names = self.sampler.get_blobs().flatten()[0].keys() self.samples = np.arange(0, blobs.__len__()) From 056b7f5ea8e15411b8723afac66d659a144d76f5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 11:53:30 +0100 Subject: [PATCH 139/288] nsamples kwarg added to sample_from_high_density_region --- indica/workflows/bayes_workflow_example.py | 25 ++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 5d2e5868..a6a2ad0d 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -6,7 +6,7 @@ from indica.bayesmodels import BayesModels, get_uniform from indica.workflows.bayes_plots import plot_bayes_result from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow -from indica.writers.bda_tree import create_nodes, write_nodes, check_analysis_run +from indica.writers.bda_tree import create_nodes, write_nodes from indica.converters.line_of_sight import LineOfSightTransform from indica.models.interferometry import Interferometry, smmh1_transform_example @@ -99,7 +99,12 @@ "Ti_prof.wcenter", "Ti_prof.peaking", ] - +OPTIMISED_QUANTITY = [ + "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne", + ] class BayesWorkflowExample(AbstractBayesWorkflow): def __init__( @@ -330,13 +335,11 @@ def _exp_data(self, **kwargs): def setup_optimiser(self, model_kwargs, - iterations=200, nwalkers=50, burn_frac=0.10, sample_high_density=False, ): self.model_kwargs = model_kwargs - self.iterations = iterations self.nwalkers = nwalkers self.burn_frac = burn_frac self.sample_high_density = sample_high_density @@ -352,7 +355,7 @@ def setup_optimiser(self, ndim = len(self.param_names) self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] self.sampler = emcee.EnsembleSampler( - self.nwalkers, + nwalkers, ndim, log_prob_fn=self.bayesmodel.ln_posterior, parameter_names=self.param_names, @@ -366,7 +369,7 @@ def setup_optimiser(self, def _sample_start_points(self, sample_high_density: bool = True): if sample_high_density: start_points = self.bayesmodel.sample_from_high_density_region( - self.param_names, self.sampler, self.nwalkers + self.param_names, self.sampler, self.nwalkers, nsamples=100 ) else: start_points = self.bayesmodel.sample_from_priors( @@ -374,8 +377,8 @@ def _sample_start_points(self, sample_high_density: bool = True): ) return start_points - def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_to_write=None, plot=False, - **kwargs): + def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_to_write=None, + plot=False, iterations=100, burn_frac=0.10, **kwargs): if mds_write: # check_analysis_run(self.pulse, self.run) self.node_structure = create_nodes( @@ -385,7 +388,7 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ mode="NEW", ) - self.run_sampler() + self.run_sampler(iterations=iterations, burn_frac=burn_frac) self.save_pickle(filepath=filepath) if plot: # currently requires result with DataArrays @@ -422,9 +425,7 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ ) run.setup_opt_data(phantoms=run.phantoms) run.setup_optimiser( - iterations=100, nwalkers=50, - burn_frac=0.10, sample_high_density=True, model_kwargs={ } @@ -435,4 +436,6 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ run="RUN01", mds_write=True, plot=True, + burn_frac=0.10, + iterations=100, ) From e1b35c537ac54a308b4fca77ad9f2ad4eb794e53 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 11:53:54 +0100 Subject: [PATCH 140/288] mocked bda_tree module --- .../workflows/test_bayes_workflow_example.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index b80fc392..6dd2370c 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -1,4 +1,7 @@ import pytest +from unittest.mock import Mock +import sys + import copy import numpy as np @@ -10,11 +13,16 @@ OPTIMISED_QUANTITY, ) + + + """ TODO: Mock reader for testing experimental data reading """ +sys.modules["indica.writers.bda_tree"] = Mock() + class TestBayesWorkflowExample: def setup_class(self): @@ -39,9 +47,7 @@ def setup_class(self): model_kwargs = { "xrcs_moment_analysis": False, }, - iterations = 1, nwalkers = 20, - burn_frac = 0.10, sample_high_density = False, ) @@ -50,7 +56,13 @@ def setup_class(self): pulse_to_write=23000101, run="RUN01", mds_write = False, - plot = False,) + plot = False, + iterations=1, + burn_frac=0.10, + ) + self.sampler_settings = dict( + iterations=1, + burn_frac=0.10,) self.workflow_untouched = BayesWorkflowExample(**self.init_settings) self.workflow=None @@ -147,7 +159,7 @@ def test_worklow_has_results_after_run(self): self.workflow.setup_plasma(**self.plasma_settings) self.workflow.setup_opt_data(phantoms=True) self.workflow.setup_optimiser(**self.optimiser_settings) - self.workflow.run_sampler() + self.workflow.run_sampler(**self.sampler_settings) if not hasattr(self.workflow, "result"): raise ValueError(f"missing result in workflow object") assert True From 3733bc6f03fda0ef44198fac82323320ae5b282f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 11:56:36 +0100 Subject: [PATCH 141/288] fixed import --- indica/workflows/load_modelling_plasma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/load_modelling_plasma.py b/indica/workflows/load_modelling_plasma.py index e0fcd6a5..d0e0a5fe 100644 --- a/indica/workflows/load_modelling_plasma.py +++ b/indica/workflows/load_modelling_plasma.py @@ -13,7 +13,7 @@ from indica.equilibrium import Equilibrium from indica.models.charge_exchange import ChargeExchange from indica.models.diode_filters import BremsstrahlungDiode -from indica.models.helike_spectroscopy import Helike_spectroscopy +from indica.models.helike_spectroscopy import HelikeSpectrometer from indica.models.interferometry import Interferometry from indica.models.plasma import Plasma from indica.models.sxr_camera import SXRcamera From b0f3fae648bfae748ae40267ddf407c5407a08b4 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 11:56:54 +0100 Subject: [PATCH 142/288] fixed import --- indica/workflows/load_modelling_plasma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/load_modelling_plasma.py b/indica/workflows/load_modelling_plasma.py index d0e0a5fe..957577bc 100644 --- a/indica/workflows/load_modelling_plasma.py +++ b/indica/workflows/load_modelling_plasma.py @@ -31,7 +31,7 @@ DIAGNOSTIC_MODELS = { "smmh1": Interferometry, "nirh1": Interferometry, - "xrcs": Helike_spectroscopy, + "xrcs": HelikeSpectrometer, "ts": ThomsonScattering, "cxff_pi": ChargeExchange, "cxff_tws_c": ChargeExchange, From ad9400a8053a2dbd03376214741189c7cb5a61f3 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 11:58:12 +0100 Subject: [PATCH 143/288] black formatting --- indica/workflows/abstract_bayes_workflow.py | 128 ++++++++++--------- indica/workflows/bayes_workflow_example.py | 132 ++++++++++---------- indica/workflows/profile_workflows.py | 1 - indica/workflows/run_tomo_1d.py | 2 - indica/workflows/zeff_profile.py | 8 +- 5 files changed, 132 insertions(+), 139 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 0f8afcfc..a8496855 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -12,13 +12,13 @@ class AbstractBayesWorkflow(ABC): @abstractmethod def __init__( - self, - pulse=None, - phantoms=None, - diagnostics=None, - param_names=None, - opt_quantity=None, - priors=None, + self, + pulse=None, + phantoms=None, + diagnostics=None, + param_names=None, + opt_quantity=None, + priors=None, ): self.pulse = pulse self.phantoms = phantoms @@ -111,8 +111,8 @@ def save_phantom_profiles(self, kinetic_profiles=None): if self.phantoms: phantom_profiles = { profile_key: getattr(self.plasma, profile_key) - .sel(t=self.plasma.time_to_calculate) - .copy() + .sel(t=self.plasma.time_to_calculate) + .copy() for profile_key in kinetic_profiles } else: @@ -120,7 +120,7 @@ def save_phantom_profiles(self, kinetic_profiles=None): profile_key: getattr(self.plasma, profile_key).sel( t=self.plasma.time_to_calculate ) - * 0 + * 0 for profile_key in kinetic_profiles } @@ -225,22 +225,22 @@ def _build_result_dict(self): "RHO_TOR": self.plasma.equilibrium.rhotor.interp(t=self.plasma.t), "NE": self.blobs["electron_density"].median(dim="index"), "NI": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .median(dim="index"), "TE": self.blobs["electron_temperature"].median(dim="index"), "TI": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .median(dim="index"), "NFAST": self.blobs["fast_density"].median(dim="index"), "NNEUTR": self.blobs["neutral_density"].median(dim="index"), "NE_ERR": self.blobs["electron_density"].std(dim="index"), "NI_ERR": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .std(dim="index"), "TE_ERR": self.blobs["electron_temperature"].std(dim="index"), "TI_ERR": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .std(dim="index"), "NFAST_ERR": self.blobs["fast_density"].std(dim="index"), "NNEUTR_ERR": self.blobs["neutral_density"].std(dim="index"), } @@ -248,8 +248,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"NIZ{num_imp + 1}": self.blobs["impurity_density"] - .sel(element=imp) - .median(dim="index") + .sel(element=imp) + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -257,8 +257,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"NIZ{num_imp + 1}_ERR": self.blobs["impurity_density"] - .sel(element=imp) - .std(dim="index") + .sel(element=imp) + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -266,8 +266,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"TIZ{num_imp + 1}": self.blobs["ion_temperature"] - .sel(element=imp) - .median(dim="index") + .sel(element=imp) + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -275,8 +275,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"TIZ{num_imp + 1}_ERR": self.blobs["ion_temperature"] - .sel(element=imp) - .std(dim="index") + .sel(element=imp) + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -315,41 +315,41 @@ def _build_result_dict(self): result["GLOBAL"] = { "TI0": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "TE0": self.blobs["electron_temperature"] - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "NE0": self.blobs["electron_density"] - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "NI0": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "TI0_ERR": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), "TE0_ERR": self.blobs["electron_temperature"] - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), "NE0_ERR": self.blobs["electron_density"] - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), "NI0_ERR": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index"), } result["GLOBAL"] = { **result["GLOBAL"], **{ f"TI0Z{num_imp + 1}": self.blobs["ion_temperature"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -357,9 +357,9 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"TI0Z{num_imp + 1}_ERR": self.blobs["ion_temperature"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -367,9 +367,9 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"NI0Z{num_imp + 1}": self.blobs["impurity_density"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -377,9 +377,9 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"NI0Z{num_imp + 1}_ERR": self.blobs["impurity_density"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -408,9 +408,7 @@ def run_sampler(self, iterations, burn_frac): self.param_names.__len__(), auto_sample=10, ) - blobs = self.sampler.get_blobs( - discard=int(iterations * burn_frac), flat=True - ) + blobs = self.sampler.get_blobs(discard=int(iterations * burn_frac), flat=True) blob_names = self.sampler.get_blobs().flatten()[0].keys() self.samples = np.arange(0, blobs.__len__()) @@ -459,9 +457,9 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl autocorr = np.ones(shape=(iterations, n_params)) * np.nan old_tau = np.inf for sample in sampler.sample( - start_points, - iterations=iterations, - progress=True, + start_points, + iterations=iterations, + progress=True, ): if sampler.iteration % auto_sample: continue @@ -472,5 +470,5 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl if converged: break old_tau = new_tau - autocorr = autocorr[: sampler.iteration, ] + autocorr = autocorr[: sampler.iteration,] return autocorr diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index a6a2ad0d..eebc3d82 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -50,7 +50,6 @@ "Ne_prof.wped": get_uniform(1, 6), "Ne_prof.wcenter": get_uniform(0.1, 0.8), "Ne_prof.peaking": get_uniform(1, 6), - "Nimp_prof.y0": loguniform(1e16, 1e18), "Nimp_prof.y1": get_uniform(1e15, 1e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( @@ -63,12 +62,10 @@ "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where( (x1 > x2), 1, 0 ), # impurity always more peaked - "Te_prof.y0": get_uniform(1000, 5000), "Te_prof.wped": get_uniform(1, 6), "Te_prof.wcenter": get_uniform(0.1, 0.8), "Te_prof.peaking": get_uniform(1, 10), - # "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode "Ti_prof.y0": get_uniform(1000, 10000), "Ti_prof.wped": get_uniform(1, 6), @@ -82,43 +79,41 @@ # "Ne_prof.peaking", # "Ne_prof.wcenter", # "Ne_prof.wped", - # "Nimp_prof.y1", "Nimp_prof.y0", # "Nimp_prof.wcenter", # "Nimp_prof.wped", # "Nimp_prof.peaking", - "Te_prof.y0", # "Te_prof.wped", "Te_prof.wcenter", "Te_prof.peaking", - "Ti_prof.y0", # "Ti_prof.wped", "Ti_prof.wcenter", "Ti_prof.peaking", ] OPTIMISED_QUANTITY = [ - "xrcs.spectra", - "cxff_pi.ti", - "efit.wp", - "smmh1.ne", - ] + "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne", +] + class BayesWorkflowExample(AbstractBayesWorkflow): def __init__( - self, - pulse: int = None, - diagnostics: list = None, - param_names: list = None, - opt_quantity: list = None, - priors: dict = None, - profile_params: dict = None, - phantoms: bool = False, - tstart=0.02, - tend=0.10, - dt=0.005, + self, + pulse: int = None, + diagnostics: list = None, + param_names: list = None, + opt_quantity: list = None, + priors: dict = None, + profile_params: dict = None, + phantoms: bool = False, + tstart=0.02, + tend=0.10, + dt=0.005, ): self.pulse = pulse self.diagnostics = diagnostics @@ -160,18 +155,18 @@ def __init__( self.read_data(self.diagnostics) self.setup_models(self.diagnostics) - def setup_plasma(self, - tstart=None, - tend=None, - dt=None, - tsample=0.050, - main_ion = "h", - impurities = ("ar", "c"), - impurity_concentration = (0.001, 0.04), - n_rad=20, - **kwargs - ): - + def setup_plasma( + self, + tstart=None, + tend=None, + dt=None, + tsample=0.050, + main_ion="h", + impurities=("ar", "c"), + impurity_concentration=(0.001, 0.04), + n_rad=20, + **kwargs, + ): if not all([tstart, tend, dt]): tstart = self.tstart tend = self.tend @@ -227,9 +222,7 @@ def setup_models(self, diagnostics: list): window = None if hasattr(self, "data"): if diag in self.data.keys(): - window = ( - self.data[diag]["spectra"].wavelength.values - ) + window = self.data[diag]["spectra"].wavelength.values model = HelikeSpectrometer( name="xrcs", @@ -266,28 +259,28 @@ def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): if "smmh1" in self.diagnostics: opt_data["smmh1.ne"] = ( self.models["smmh1"]() - .pop("ne") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("ne") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if "xrcs" in self.diagnostics: opt_data["xrcs.spectra"] = ( self.models["xrcs"]() - .pop("spectra") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("spectra") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) opt_data["xrcs.spectra"]["error"] = np.sqrt(opt_data["xrcs.spectra"]) if "cxff_pi" in self.diagnostics: cxrs_data = ( self.models["cxff_pi"]() - .pop("ti") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("ti") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2, drop=True) if "efit" in self.diagnostics: opt_data["efit.wp"] = ( self.models["efit"]() - .pop("wp") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) + .pop("wp") + .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) if noise: @@ -313,7 +306,7 @@ def _exp_data(self, **kwargs): opt_data = flatdict.FlatDict(self.data, ".") if "xrcs" in self.diagnostics: opt_data["xrcs.spectra"]["wavelength"] = ( - opt_data["xrcs.spectra"].wavelength * 0.1 + opt_data["xrcs.spectra"].wavelength * 0.1 ) background = opt_data["xrcs.spectra"].where( (opt_data["xrcs.spectra"].wavelength < 0.392) @@ -333,12 +326,13 @@ def _exp_data(self, **kwargs): ) return opt_data - def setup_optimiser(self, - model_kwargs, - nwalkers=50, - burn_frac=0.10, - sample_high_density=False, - ): + def setup_optimiser( + self, + model_kwargs, + nwalkers=50, + burn_frac=0.10, + sample_high_density=False, + ): self.model_kwargs = model_kwargs self.nwalkers = nwalkers self.burn_frac = burn_frac @@ -377,8 +371,17 @@ def _sample_start_points(self, sample_high_density: bool = True): ) return start_points - def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_to_write=None, - plot=False, iterations=100, burn_frac=0.10, **kwargs): + def __call__( + self, + filepath="./results/test/", + run=None, + mds_write=False, + pulse_to_write=None, + plot=False, + iterations=100, + burn_frac=0.10, + **kwargs, + ): if mds_write: # check_analysis_run(self.pulse, self.run) self.node_structure = create_nodes( @@ -405,13 +408,13 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ run = BayesWorkflowExample( pulse=None, phantoms=True, - diagnostics=["xrcs", "efit", "smmh1","cxff_pi"], + diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], opt_quantity=[ - "xrcs.spectra", - "cxff_pi.ti", - "efit.wp", - "smmh1.ne", - ], + "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne", + ], param_names=OPTIMISED_PARAMS, profile_params=DEFAULT_PROFILE_PARAMS, priors=DEFAULT_PRIORS, @@ -421,15 +424,10 @@ def __call__(self, filepath="./results/test/", run=None, mds_write=False, pulse_ ) run.setup_plasma( - tsample=0.060, + tsample=0.060, ) run.setup_opt_data(phantoms=run.phantoms) - run.setup_optimiser( - nwalkers=50, - sample_high_density=True, - model_kwargs={ - } - ) + run.setup_optimiser(nwalkers=50, sample_high_density=True, model_kwargs={}) results = run( filepath="./results/test/", pulse_to_write=23000101, diff --git a/indica/workflows/profile_workflows.py b/indica/workflows/profile_workflows.py index bf072059..27c4e292 100644 --- a/indica/workflows/profile_workflows.py +++ b/indica/workflows/profile_workflows.py @@ -60,7 +60,6 @@ def profile_scans_pca( save_fig: bool = False, fig_path: str = None, ): - if fig_path is None: fig_path = FIG_PATH diff --git a/indica/workflows/run_tomo_1d.py b/indica/workflows/run_tomo_1d.py index 5398c5db..c4a73fa3 100644 --- a/indica/workflows/run_tomo_1d.py +++ b/indica/workflows/run_tomo_1d.py @@ -201,7 +201,6 @@ def sxrc_xy( save_fig: bool = True, instrument="sxrc_xy2", ): - if input_dict is None: st40 = ReadST40(pulse, tstart, tend, dt=dt) st40(instruments=[instrument], map_diagnostics=False) @@ -298,7 +297,6 @@ def old_camera( reg_level_guess: float = 0.5, input_dict: dict = None, ): - if input_dict is None: st40 = ReadST40(pulse, tstart, tend) # return st40 diff --git a/indica/workflows/zeff_profile.py b/indica/workflows/zeff_profile.py index dd88f85d..a87ac99b 100644 --- a/indica/workflows/zeff_profile.py +++ b/indica/workflows/zeff_profile.py @@ -196,7 +196,10 @@ def prepare_data_cxrs( continue data = st40.binned_data[instrument] attrs = data["spectra"].attrs - (data["background"], data["brightness"],) = models[ + ( + data["background"], + data["brightness"], + ) = models[ instrument ].integrate_spectra(data["spectra"]) data["background"].attrs = attrs @@ -222,7 +225,6 @@ def prepare_inputs( phantom_data: bool = True, ts_side: str = "LFS", ): - flat_data: dict = {} models: dict = {} @@ -350,7 +352,6 @@ def run_bayes( phantom_data: bool = True, ts_side: str = "LFS", ): - plasma, models, flat_data, input_profiles = prepare_inputs( pulse, tstart=tstart, @@ -430,7 +431,6 @@ def run_inversion( phantom_data: bool = True, ts_side: str = "LFS", ): - plasma, models, flat_data, input_profiles = prepare_inputs( pulse, tstart=tstart, From f0a1d658ff9fbc635f5750048a9cb79343e331f1 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 11:59:18 +0100 Subject: [PATCH 144/288] black formatting --- .../workflows/test_bayes_workflow_example.py | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index 6dd2370c..a389d381 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -14,8 +14,6 @@ ) - - """ TODO: Mock reader for testing experimental data reading @@ -28,7 +26,7 @@ class TestBayesWorkflowExample: def setup_class(self): self.init_settings = dict( pulse=None, - phantoms = True, + phantoms=True, diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], opt_quantity=OPTIMISED_QUANTITY, param_names=OPTIMISED_PARAMS, @@ -39,33 +37,33 @@ def setup_class(self): dt=0.005, ) self.plasma_settings = dict( - tsample=0.060, + tsample=0.060, ) - self.optimiser_settings = dict( - model_kwargs = { - "xrcs_moment_analysis": False, - }, - nwalkers = 20, - sample_high_density = False, + model_kwargs={ + "xrcs_moment_analysis": False, + }, + nwalkers=20, + sample_high_density=False, ) self.call_settings = dict( filepath=None, pulse_to_write=23000101, run="RUN01", - mds_write = False, - plot = False, + mds_write=False, + plot=False, iterations=1, burn_frac=0.10, ) self.sampler_settings = dict( iterations=1, - burn_frac=0.10,) + burn_frac=0.10, + ) self.workflow_untouched = BayesWorkflowExample(**self.init_settings) - self.workflow=None + self.workflow = None def setup_method(self): self.workflow = copy.deepcopy(self.workflow_untouched) @@ -74,12 +72,7 @@ def teardown_method(self): self.workflow = None def test_workflow_initializes(self): - attributes_to_check = [ - "data", - "reader", - "models", - "equilibrium" - ] + attributes_to_check = ["data", "reader", "models", "equilibrium"] for attribute in attributes_to_check: if not hasattr(self.workflow, attribute): raise ValueError(f"missing {attribute} in workflow object") @@ -145,14 +138,22 @@ def test_phantom_data_runs_with_noise_added(self): def test_sampling_from_priors(self): self.workflow.setup_plasma(**self.plasma_settings) - self.workflow.setup_opt_data(phantoms=True, ) - self.workflow.setup_optimiser(**dict(self.optimiser_settings, **{"sample_high_density":False})) + self.workflow.setup_opt_data( + phantoms=True, + ) + self.workflow.setup_optimiser( + **dict(self.optimiser_settings, **{"sample_high_density": False}) + ) assert True def test_sampling_from_high_density(self): self.workflow.setup_plasma(**self.plasma_settings) - self.workflow.setup_opt_data(phantoms=True, ) - self.workflow.setup_optimiser(**dict(self.optimiser_settings, **{"sample_high_density":True})) + self.workflow.setup_opt_data( + phantoms=True, + ) + self.workflow.setup_optimiser( + **dict(self.optimiser_settings, **{"sample_high_density": True}) + ) assert True def test_worklow_has_results_after_run(self): From 3b1dcd8d4d9865a6aedf18777f7606889c2e01bc Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 13:18:57 +0100 Subject: [PATCH 145/288] fixed where mocked module is imported --- tests/unit/workflows/test_bayes_workflow_example.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index a389d381..6db6e361 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -4,6 +4,7 @@ import copy import numpy as np +sys.modules["indica.writers.bda_tree"] = Mock() from indica.workflows.bayes_workflow_example import ( BayesWorkflowExample, @@ -19,8 +20,6 @@ Mock reader for testing experimental data reading """ -sys.modules["indica.writers.bda_tree"] = Mock() - class TestBayesWorkflowExample: def setup_class(self): From b8b6ceebca889385bc85366d7d79ce9fe33e9ada Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 13:27:36 +0100 Subject: [PATCH 146/288] black formatting --- indica/models/helike_spectroscopy.py | 2 ++ indica/models/interferometry.py | 5 ++--- indica/models/plasma.py | 8 +++++++- tests/unit/workflows/test_bayes_workflow_example.py | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 5fd32aae..90ac5c4e 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -465,6 +465,7 @@ def __call__( self._build_bckc_dictionary() return self.bckc + def helike_transform_example(nchannels): los_end = np.full((nchannels, 3), 0.0) los_end[:, 0] = 0.17 @@ -488,6 +489,7 @@ def helike_transform_example(nchannels): ) return los_transform + def example_run( pulse: int = None, plasma=None, plot=False, moment_analysis: bool = False, **kwargs ): diff --git a/indica/models/interferometry.py b/indica/models/interferometry.py index c9354205..5a7cf49b 100644 --- a/indica/models/interferometry.py +++ b/indica/models/interferometry.py @@ -90,9 +90,8 @@ def __call__( def smmh1_transform_example(nchannels): - - los_start = np.array([[0.8, 0, 0]]) * np.ones((nchannels,3)) - los_start[:,2] = np.linspace(0, -0.2, nchannels) + los_start = np.array([[0.8, 0, 0]]) * np.ones((nchannels, 3)) + los_start[:, 2] = np.linspace(0, -0.2, nchannels) los_end = np.array([[0.17, 0, 0]]) * np.ones((nchannels, 3)) los_end[:, 2] = np.linspace(0, -0.2, nchannels) origin = los_start diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 86630796..435b2d1d 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -517,7 +517,13 @@ def update_profiles( else: raise ValueError(f"parameter: {key} not found in {prefix}") - for key in ["electron_density", "electron_temperature", "ion_temperature", "toroidal_rotation", "impurity_density"]: + for key in [ + "electron_density", + "electron_temperature", + "ion_temperature", + "toroidal_rotation", + "impurity_density", + ]: self.assign_profiles(key, t=self.time_to_calculate) @property diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index 6db6e361..faaa9347 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -4,6 +4,7 @@ import copy import numpy as np + sys.modules["indica.writers.bda_tree"] = Mock() from indica.workflows.bayes_workflow_example import ( From d3e4e5c4779cf80a20c7e657c35638cdab118a91 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 14:00:23 +0100 Subject: [PATCH 147/288] mocking bda_tree import --- indica/bayesmodels.py | 4 +++- tests/conftest.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 47685db7..900946a6 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -1,5 +1,6 @@ from copy import deepcopy import warnings + import numpy as np from scipy.stats import uniform @@ -82,7 +83,8 @@ def _build_bckc(self, params, **kwargs): """ self.bckc: dict = {} for model in self.diagnostic_models: - # removes "model.name." from params and kwargs then passes them to model e.g. xrcs.background -> background + # removes "model.name." from params and kwargs then passes them to model + # e.g. xrcs.background -> background _nuisance_params = { param_name.replace(model.name + ".", ""): param_value for param_name, param_value in params.items() diff --git a/tests/conftest.py b/tests/conftest.py index 209c8a1b..e5b9475a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,3 +25,4 @@ # Turn off import of modules that cnnot be installed in CI sys.modules["indica.readers.st40reader"] = mock.MagicMock() sys.modules["indica.readers.st40reader.ST40Reader"] = mock.MagicMock() +sys.modules["indica.writers.bda_tree"] = mock.Mock() From a7ec4cdf724588810c4a2f6984c5f5e6c80455bf Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 15:26:39 +0100 Subject: [PATCH 148/288] precommit --- indica/models/plasma.py | 1 - indica/workflows/abstract_bayes_workflow.py | 22 +++++++----- indica/workflows/bayes_plots.py | 11 +++--- indica/workflows/bayes_workflow_example.py | 32 +++++++++-------- indica/workflows/zeff_profile.py | 5 +-- indica/writers/bda_tree.py | 9 +++-- .../workflows/test_bayes_workflow_example.py | 35 +++++++------------ 7 files changed, 56 insertions(+), 59 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 435b2d1d..0c635f88 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -14,7 +14,6 @@ from indica.datatypes import ELEMENTS from indica.equilibrium import Equilibrium from indica.equilibrium import fake_equilibrium_data -from indica.numpy_typing import LabeledArray from indica.operators.atomic_data import FractionalAbundance from indica.operators.atomic_data import PowerLoss import indica.physics as ph diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index a8496855..f62c037d 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -1,12 +1,14 @@ -from abc import ABC, abstractmethod -import numpy as np -import xarray as xr -import pandas as pd +from abc import ABC +from abc import abstractmethod from pathlib import Path import pickle -from indica.readers.read_st40 import ReadST40 +import numpy as np +import pandas as pd +import xarray as xr + from indica.equilibrium import fake_equilibrium +from indica.readers.read_st40 import ReadST40 class AbstractBayesWorkflow(ABC): @@ -162,7 +164,7 @@ def _build_result_dict(self): } result["INPUT"]["WORKFLOW"] = { diag_name.upper(): { - "PULSE": self.pulse, # Change this if different pulses used for diagnostics + "PULSE": self.pulse, # Change this if different pulses used "USAGE": "".join( [quantity[1] for quantity in quant_list if quantity[0] == diag_name] ), @@ -464,11 +466,15 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl if sampler.iteration % auto_sample: continue new_tau = sampler.get_autocorr_time(tol=0) - autocorr[sampler.iteration - 1,] = new_tau + autocorr[ + sampler.iteration - 1, + ] = new_tau converged = np.all(new_tau * 50 < sampler.iteration) converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) if converged: break old_tau = new_tau - autocorr = autocorr[: sampler.iteration,] + autocorr = autocorr[ + : sampler.iteration, + ] return autocorr diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index c027c004..93955202 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -1,12 +1,13 @@ import os +import pickle import corner +import flatdict import matplotlib.pyplot as plt import numpy as np -import pickle -import flatdict -from indica.utilities import set_plot_rcparams, set_axis_sci +from indica.utilities import set_axis_sci +from indica.utilities import set_plot_rcparams def plot_profile( @@ -406,7 +407,7 @@ def plot_bayes_result( profiles[key], key, figheader=figheader, - filename=f"ar density", + filename="ar density", filetype=filetype, phantom_profile=phantom_profiles, color="red", @@ -417,7 +418,7 @@ def plot_bayes_result( profiles[key], key, figheader=figheader, - filename=f"c density", + filename="c density", filetype=filetype, phantom_profile=phantom_profiles, color="red", diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index eebc3d82..53809312 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -1,22 +1,22 @@ import emcee -import numpy as np import flatdict +import numpy as np from scipy.stats import loguniform -from indica.bayesmodels import BayesModels, get_uniform -from indica.workflows.bayes_plots import plot_bayes_result -from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow -from indica.writers.bda_tree import create_nodes, write_nodes -from indica.converters.line_of_sight import LineOfSightTransform - -from indica.models.interferometry import Interferometry, smmh1_transform_example -from indica.models.helike_spectroscopy import ( - HelikeSpectrometer, - helike_transform_example, -) -from indica.models.charge_exchange import ChargeExchange, pi_transform_example +from indica.bayesmodels import BayesModels +from indica.bayesmodels import get_uniform +from indica.models.charge_exchange import ChargeExchange +from indica.models.charge_exchange import pi_transform_example from indica.models.equilibrium_reconstruction import EquilibriumReconstruction +from indica.models.helike_spectroscopy import helike_transform_example +from indica.models.helike_spectroscopy import HelikeSpectrometer +from indica.models.interferometry import Interferometry +from indica.models.interferometry import smmh1_transform_example from indica.models.plasma import Plasma +from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow +from indica.workflows.bayes_plots import plot_bayes_result +from indica.writers.bda_tree import create_nodes +from indica.writers.bda_tree import write_nodes # global configurations DEFAULT_PROFILE_PARAMS = { @@ -289,7 +289,11 @@ def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): ].max().values * np.random.normal(0, noise_factor, None) opt_data["xrcs.spectra"] = opt_data["xrcs.spectra"] + np.random.normal( 0, - np.sqrt(opt_data["xrcs.spectra"].values[0,]), + np.sqrt( + opt_data["xrcs.spectra"].values[ + 0, + ] + ), opt_data["xrcs.spectra"].shape[1], ) opt_data["cxff_pi.ti"] = opt_data["cxff_pi.ti"] + opt_data[ diff --git a/indica/workflows/zeff_profile.py b/indica/workflows/zeff_profile.py index a87ac99b..adf29bda 100644 --- a/indica/workflows/zeff_profile.py +++ b/indica/workflows/zeff_profile.py @@ -196,10 +196,7 @@ def prepare_data_cxrs( continue data = st40.binned_data[instrument] attrs = data["spectra"].attrs - ( - data["background"], - data["brightness"], - ) = models[ + (data["background"], data["brightness"],) = models[ instrument ].integrate_spectra(data["spectra"]) data["background"].attrs = attrs diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 6f25ecb7..5a5878a8 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -1,9 +1,10 @@ # Create trees from ppac standard utility tools -import standard_utility as util -from MDSplus import Connection import sys + +from MDSplus import Connection import numpy as np +import standard_utility as util def bda(): @@ -165,8 +166,6 @@ def create_nodes( } for diag_name in diag_names } - # for diag_name in diag_names: - # diag_nodes[diag_name]["RUN"] = ("TEXT", f"RUN from which {diag_name} data was taken") nodes = { "RUN": ("TEXT", "RUN used for diagnostic"), @@ -219,7 +218,7 @@ def create_nodes( def write_nodes(pulse, node_info, data): util.StandardNodeWriting( - pulse_number=pulse, # pulse number for which data should be written to the structure + pulse_number=pulse, # pulse number for which data should be written dict_node_info=node_info, # node information file nodes_to_write=[], # selective nodes to be written data_to_write=data, diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index faaa9347..d3111cb1 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -1,20 +1,14 @@ -import pytest -from unittest.mock import Mock -import sys - import copy -import numpy as np - -sys.modules["indica.writers.bda_tree"] = Mock() +import sys +from unittest.mock import Mock -from indica.workflows.bayes_workflow_example import ( - BayesWorkflowExample, - DEFAULT_PRIORS, - DEFAULT_PROFILE_PARAMS, - OPTIMISED_PARAMS, - OPTIMISED_QUANTITY, -) +import pytest +from indica.workflows.bayes_workflow_example import BayesWorkflowExample +from indica.workflows.bayes_workflow_example import DEFAULT_PRIORS +from indica.workflows.bayes_workflow_example import DEFAULT_PROFILE_PARAMS +from indica.workflows.bayes_workflow_example import OPTIMISED_PARAMS +from indica.workflows.bayes_workflow_example import OPTIMISED_QUANTITY """ TODO: @@ -80,18 +74,15 @@ def test_workflow_initializes(self): def test_init_phantoms_false_with_example_plasma(self): with pytest.raises(ValueError): - example = BayesWorkflowExample( - dict(self.init_settings, **{"phantoms": False}) - ) + BayesWorkflowExample(dict(self.init_settings, **{"phantoms": False})) def test_init_not_including_all_required_inputs(self): with pytest.raises(ValueError): - example = BayesWorkflowExample( - dict(self.init_settings, **{"param_names": None}) - ) + BayesWorkflowExample(dict(self.init_settings, **{"param_names": None})) # def test_reader_has_read_all_diagnostic_data(self): - # assert all(diag_name in self.workflow.reader.keys() for diag_name in self.workflow.diagnostics) + # assert all(diag_name in self.workflow.reader.keys() + # for diag_name in self.workflow.diagnostics) def test_plasma_has_equilibrium(self): self.workflow.setup_plasma(**self.plasma_settings) @@ -162,7 +153,7 @@ def test_worklow_has_results_after_run(self): self.workflow.setup_optimiser(**self.optimiser_settings) self.workflow.run_sampler(**self.sampler_settings) if not hasattr(self.workflow, "result"): - raise ValueError(f"missing result in workflow object") + raise ValueError("missing result in workflow object") assert True From b1babc4ad83f6b23a60f2cffb75f2d49b889fe16 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 15:27:43 +0100 Subject: [PATCH 149/288] precommit --- tests/unit/workflows/test_bayes_workflow_example.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index d3111cb1..d725eb47 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -1,6 +1,4 @@ import copy -import sys -from unittest.mock import Mock import pytest From 8e2f2d8082a5f0d6e4763ceb22d20f07936769cd Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 15:38:35 +0100 Subject: [PATCH 150/288] moving read_data tvector to kwargs --- indica/workflows/abstract_bayes_workflow.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index f62c037d..4b7d2ef8 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -31,19 +31,19 @@ def __init__( self.read_data(self.diagnostics) self.setup_models(self.diagnostics) - def read_test_data(self, diagnostic_transforms: dict): + def read_test_data(self, diagnostic_transforms: dict, tstart=None, tend=None, dt=None): # Used with phantom data for purposes of tests print("Reading fake data") self.reader = None self.equilibrium = fake_equilibrium( - self.tstart, - self.tend, - self.dt, + tstart, + tend, + dt, ) self.transforms = diagnostic_transforms self.data = {} - def read_data(self, diagnostics: list): + def read_data(self, diagnostics: list, tstart=None, tend=None, dt=None): self.reader = ReadST40( self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt ) From 519c4eda6a8b784586ebe72d26e3f36c2a36ec80 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 15:45:01 +0100 Subject: [PATCH 151/288] precommit formatting --- indica/bayesmodels.py | 2 +- indica/workflows/abstract_bayes_workflow.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 900946a6..a05e8c77 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -194,7 +194,7 @@ def sample_from_priors(self, param_names, size=10): return samples.transpose() def sample_from_high_density_region( - self, param_names: list, sampler: object, nwalkers: int, nsamples=100 + self, param_names: list, sampler, nwalkers: int, nsamples=100 ): start_points = self.sample_from_priors(param_names, size=nsamples) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 4b7d2ef8..3e83fd4e 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -34,14 +34,13 @@ def __init__( def read_test_data(self, diagnostic_transforms: dict, tstart=None, tend=None, dt=None): # Used with phantom data for purposes of tests print("Reading fake data") - self.reader = None self.equilibrium = fake_equilibrium( tstart, tend, dt, ) self.transforms = diagnostic_transforms - self.data = {} + self.data: dict = {} def read_data(self, diagnostics: list, tstart=None, tend=None, dt=None): self.reader = ReadST40( @@ -67,7 +66,7 @@ def setup_models(self, diagnostics: list): Initialising models normally requires data to be read so transforms can be set """ - self.models = {} + self.models: dict = {} @abstractmethod def _phantom_data(self): From f7623ceebceace753bbf5738d66af10c100a5d01 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 15:50:53 +0100 Subject: [PATCH 152/288] precommit formatting --- indica/workflows/abstract_bayes_workflow.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 3e83fd4e..3221e186 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -31,7 +31,9 @@ def __init__( self.read_data(self.diagnostics) self.setup_models(self.diagnostics) - def read_test_data(self, diagnostic_transforms: dict, tstart=None, tend=None, dt=None): + def read_test_data( + self, diagnostic_transforms: dict, tstart=None, tend=None, dt=None + ): # Used with phantom data for purposes of tests print("Reading fake data") self.equilibrium = fake_equilibrium( @@ -44,7 +46,7 @@ def read_test_data(self, diagnostic_transforms: dict, tstart=None, tend=None, dt def read_data(self, diagnostics: list, tstart=None, tend=None, dt=None): self.reader = ReadST40( - self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt + self.pulse, tstart=tstart, tend=tend, dt=dt ) self.reader(diagnostics) self.equilibrium = self.reader.equilibrium From 67bd35c4e559b90031ecfff836f634e9f48519f6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 16:02:04 +0100 Subject: [PATCH 153/288] precommit types --- indica/workflows/bayes_workflow_example.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 53809312..bce9272c 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -2,6 +2,7 @@ import flatdict import numpy as np from scipy.stats import loguniform +from typing import Optional from indica.bayesmodels import BayesModels from indica.bayesmodels import get_uniform @@ -105,9 +106,9 @@ class BayesWorkflowExample(AbstractBayesWorkflow): def __init__( self, pulse: int = None, - diagnostics: list = None, - param_names: list = None, - opt_quantity: list = None, + diagnostics: Optional[list] = None, + param_names: Optional[list] = None, + opt_quantity: Optional[list] = None, priors: dict = None, profile_params: dict = None, phantoms: bool = False, @@ -144,15 +145,18 @@ def __init__( # TODO: Add some abstraction here if pulse is None: print("Running in test mode") + example_transforms = { + "xrcs": helike_transform_example(1), + "smmh1": smmh1_transform_example(1), + "cxff_pi": pi_transform_example(5), + } self.read_test_data( - { - "xrcs": helike_transform_example(1), - "smmh1": smmh1_transform_example(1), - "cxff_pi": pi_transform_example(5), - } + example_transforms, tstart=self.tstart, tend=self.tend, dt=self.dt ) else: - self.read_data(self.diagnostics) + self.read_data( + self.diagnostics, tstart=self.tstart, tend=self.tend, dt=self.dt + ) self.setup_models(self.diagnostics) def setup_plasma( From d1afc8720a7be7ff60de1607a93d6b0ff64f3dce Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 16:10:26 +0100 Subject: [PATCH 154/288] precommit types --- indica/workflows/bayes_workflow_example.py | 28 ++++++++++++---------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index bce9272c..5d153a01 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -1,8 +1,9 @@ +from typing import Optional + import emcee import flatdict import numpy as np from scipy.stats import loguniform -from typing import Optional from indica.bayesmodels import BayesModels from indica.bayesmodels import get_uniform @@ -105,12 +106,12 @@ class BayesWorkflowExample(AbstractBayesWorkflow): def __init__( self, + diagnostics: list, + param_names: list, + opt_quantity: list, + priors: dict, + profile_params: dict, pulse: int = None, - diagnostics: Optional[list] = None, - param_names: Optional[list] = None, - opt_quantity: Optional[list] = None, - priors: dict = None, - profile_params: dict = None, phantoms: bool = False, tstart=0.02, tend=0.10, @@ -414,18 +415,19 @@ def __call__( if __name__ == "__main__": run = BayesWorkflowExample( - pulse=None, - phantoms=True, - diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], - opt_quantity=[ + ["xrcs", "efit", "smmh1", "cxff_pi"], + [ "xrcs.spectra", "cxff_pi.ti", "efit.wp", "smmh1.ne", ], - param_names=OPTIMISED_PARAMS, - profile_params=DEFAULT_PROFILE_PARAMS, - priors=DEFAULT_PRIORS, + OPTIMISED_PARAMS, + DEFAULT_PROFILE_PARAMS, + DEFAULT_PRIORS, + + pulse=None, + phantoms=True, tstart=0.02, tend=0.10, dt=0.005, From 1f38f27fe762da26a978326376be13fcf40faffa Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 16:23:04 +0100 Subject: [PATCH 155/288] precommit types --- indica/workflows/abstract_bayes_workflow.py | 12 +++------ indica/workflows/bayes_workflow_example.py | 29 +++++++++------------ 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 3221e186..485bc768 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -45,9 +45,7 @@ def read_test_data( self.data: dict = {} def read_data(self, diagnostics: list, tstart=None, tend=None, dt=None): - self.reader = ReadST40( - self.pulse, tstart=tstart, tend=tend, dt=dt - ) + self.reader = ReadST40(self.pulse, tstart=tstart, tend=tend, dt=dt) self.reader(diagnostics) self.equilibrium = self.reader.equilibrium self.transforms = self.reader.transforms @@ -467,15 +465,11 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl if sampler.iteration % auto_sample: continue new_tau = sampler.get_autocorr_time(tol=0) - autocorr[ - sampler.iteration - 1, - ] = new_tau + autocorr[sampler.iteration - 1,] = new_tau converged = np.all(new_tau * 50 < sampler.iteration) converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) if converged: break old_tau = new_tau - autocorr = autocorr[ - : sampler.iteration, - ] + autocorr = autocorr[: sampler.iteration,] return autocorr diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 5d153a01..27a025dc 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -1,5 +1,3 @@ -from typing import Optional - import emcee import flatdict import numpy as np @@ -218,8 +216,9 @@ def setup_models(self, diagnostics: list): # passes=2, # ) los_transform.set_equilibrium(self.equilibrium) - model = Interferometry(name=diag) - model.set_los_transform(los_transform) + smmh = Interferometry(name=diag) + smmh.set_los_transform(los_transform) + self.models[diag] = smmh elif diag == "xrcs": los_transform = self.transforms[diag] @@ -229,24 +228,27 @@ def setup_models(self, diagnostics: list): if diag in self.data.keys(): window = self.data[diag]["spectra"].wavelength.values - model = HelikeSpectrometer( + xrcs = HelikeSpectrometer( name="xrcs", window_masks=[slice(0.394, 0.396)], window=window, ) - model.set_los_transform(los_transform) + xrcs.set_los_transform(los_transform) + self.models[diag] = xrcs elif diag == "efit": - model = EquilibriumReconstruction(name="efit") + efit = EquilibriumReconstruction(name="efit") + self.models[diag] = efit elif diag == "cxff_pi": transform = self.transforms[diag] transform.set_equilibrium(self.equilibrium) - model = ChargeExchange(name=diag, element="ar") - model.set_transect_transform(transform) + cxrs = ChargeExchange(name=diag, element="ar") + cxrs.set_transect_transform(transform) + self.models[diag] = cxrs + else: raise ValueError(f"{diag} not found in setup_models") - self.models[diag] = model def setup_opt_data(self, phantoms=False, **kwargs): if not hasattr(self, "plasma"): @@ -294,11 +296,7 @@ def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): ].max().values * np.random.normal(0, noise_factor, None) opt_data["xrcs.spectra"] = opt_data["xrcs.spectra"] + np.random.normal( 0, - np.sqrt( - opt_data["xrcs.spectra"].values[ - 0, - ] - ), + np.sqrt(opt_data["xrcs.spectra"].values[0,]), opt_data["xrcs.spectra"].shape[1], ) opt_data["cxff_pi.ti"] = opt_data["cxff_pi.ti"] + opt_data[ @@ -425,7 +423,6 @@ def __call__( OPTIMISED_PARAMS, DEFAULT_PROFILE_PARAMS, DEFAULT_PRIORS, - pulse=None, phantoms=True, tstart=0.02, From c555f6b6900ae5dca3e448dcf785e8b1cb5ada40 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 16:27:54 +0100 Subject: [PATCH 156/288] precommit types --- indica/workflows/bayes_workflow_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 27a025dc..a73b0cef 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -234,7 +234,7 @@ def setup_models(self, diagnostics: list): window=window, ) xrcs.set_los_transform(los_transform) - self.models[diag] = xrcs + # self.models[diag] = xrcs elif diag == "efit": efit = EquilibriumReconstruction(name="efit") From b5d584010984158a27a43d004510a94d8f6dd158 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 16:37:13 +0100 Subject: [PATCH 157/288] precommit types --- indica/workflows/bayes_workflow_example.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index a73b0cef..8e511f5e 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -2,6 +2,7 @@ import flatdict import numpy as np from scipy.stats import loguniform +from typing import Dict, Any from indica.bayesmodels import BayesModels from indica.bayesmodels import get_uniform @@ -196,7 +197,7 @@ def setup_plasma( self.save_phantom_profiles() def setup_models(self, diagnostics: list): - self.models = {} + self.models: Dict[str, Any] = {} for diag in diagnostics: if diag == "smmh1": los_transform = self.transforms[diag] @@ -216,9 +217,8 @@ def setup_models(self, diagnostics: list): # passes=2, # ) los_transform.set_equilibrium(self.equilibrium) - smmh = Interferometry(name=diag) - smmh.set_los_transform(los_transform) - self.models[diag] = smmh + model = Interferometry(name=diag) + model.set_los_transform(los_transform) elif diag == "xrcs": los_transform = self.transforms[diag] @@ -228,27 +228,24 @@ def setup_models(self, diagnostics: list): if diag in self.data.keys(): window = self.data[diag]["spectra"].wavelength.values - xrcs = HelikeSpectrometer( + model = HelikeSpectrometer( name="xrcs", window_masks=[slice(0.394, 0.396)], window=window, ) - xrcs.set_los_transform(los_transform) - # self.models[diag] = xrcs + model.set_los_transform(los_transform) elif diag == "efit": efit = EquilibriumReconstruction(name="efit") - self.models[diag] = efit elif diag == "cxff_pi": transform = self.transforms[diag] transform.set_equilibrium(self.equilibrium) - cxrs = ChargeExchange(name=diag, element="ar") - cxrs.set_transect_transform(transform) - self.models[diag] = cxrs - + model = ChargeExchange(name=diag, element="ar") + model.set_transect_transform(transform) else: raise ValueError(f"{diag} not found in setup_models") + self.models[diag] = model def setup_opt_data(self, phantoms=False, **kwargs): if not hasattr(self, "plasma"): From 9d12492cac32640d15b3277e04b3c74bc249af38 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 16:44:27 +0100 Subject: [PATCH 158/288] precommit types --- indica/workflows/bayes_workflow_example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 8e511f5e..ed1a15eb 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -198,6 +198,7 @@ def setup_plasma( def setup_models(self, diagnostics: list): self.models: Dict[str, Any] = {} + model: Any = None for diag in diagnostics: if diag == "smmh1": los_transform = self.transforms[diag] @@ -236,7 +237,7 @@ def setup_models(self, diagnostics: list): model.set_los_transform(los_transform) elif diag == "efit": - efit = EquilibriumReconstruction(name="efit") + model = EquilibriumReconstruction(name="efit") elif diag == "cxff_pi": transform = self.transforms[diag] From 15c4aacac7ed989b6ef030d4cc347743b0d98509 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 16:48:29 +0100 Subject: [PATCH 159/288] precommit types --- indica/workflows/abstract_bayes_workflow.py | 8 ++++++-- indica/workflows/bayes_workflow_example.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 485bc768..4839a2b5 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -465,11 +465,15 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl if sampler.iteration % auto_sample: continue new_tau = sampler.get_autocorr_time(tol=0) - autocorr[sampler.iteration - 1,] = new_tau + autocorr[ + sampler.iteration - 1, + ] = new_tau converged = np.all(new_tau * 50 < sampler.iteration) converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) if converged: break old_tau = new_tau - autocorr = autocorr[: sampler.iteration,] + autocorr = autocorr[ + : sampler.iteration, + ] return autocorr diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index ed1a15eb..c52bbf23 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -1,8 +1,10 @@ +from typing import Any +from typing import Dict + import emcee import flatdict import numpy as np from scipy.stats import loguniform -from typing import Dict, Any from indica.bayesmodels import BayesModels from indica.bayesmodels import get_uniform @@ -294,7 +296,11 @@ def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): ].max().values * np.random.normal(0, noise_factor, None) opt_data["xrcs.spectra"] = opt_data["xrcs.spectra"] + np.random.normal( 0, - np.sqrt(opt_data["xrcs.spectra"].values[0,]), + np.sqrt( + opt_data["xrcs.spectra"].values[ + 0, + ] + ), opt_data["xrcs.spectra"].shape[1], ) opt_data["cxff_pi.ti"] = opt_data["cxff_pi.ti"] + opt_data[ From 4362cd5bf7745f8750c93762092c3735415aafde Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 17 Aug 2023 17:05:07 +0100 Subject: [PATCH 160/288] moving kwargs/args around --- indica/workflows/bayes_workflow_example.py | 15 +++++---------- .../unit/workflows/test_bayes_workflow_example.py | 6 +++--- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index c52bbf23..cb81a09d 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -417,17 +417,12 @@ def __call__( if __name__ == "__main__": run = BayesWorkflowExample( - ["xrcs", "efit", "smmh1", "cxff_pi"], - [ - "xrcs.spectra", - "cxff_pi.ti", - "efit.wp", - "smmh1.ne", - ], - OPTIMISED_PARAMS, - DEFAULT_PROFILE_PARAMS, - DEFAULT_PRIORS, pulse=None, + diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], + param_names=OPTIMISED_PARAMS, + opt_quantity=OPTIMISED_QUANTITY, + priors=DEFAULT_PRIORS, + profile_params=DEFAULT_PROFILE_PARAMS, phantoms=True, tstart=0.02, tend=0.10, diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow_example.py index d725eb47..383f19c8 100644 --- a/tests/unit/workflows/test_bayes_workflow_example.py +++ b/tests/unit/workflows/test_bayes_workflow_example.py @@ -64,7 +64,7 @@ def teardown_method(self): self.workflow = None def test_workflow_initializes(self): - attributes_to_check = ["data", "reader", "models", "equilibrium"] + attributes_to_check = ["data", "models", "equilibrium"] for attribute in attributes_to_check: if not hasattr(self.workflow, attribute): raise ValueError(f"missing {attribute} in workflow object") @@ -72,11 +72,11 @@ def test_workflow_initializes(self): def test_init_phantoms_false_with_example_plasma(self): with pytest.raises(ValueError): - BayesWorkflowExample(dict(self.init_settings, **{"phantoms": False})) + BayesWorkflowExample(**dict(self.init_settings, **{"phantoms": False})) def test_init_not_including_all_required_inputs(self): with pytest.raises(ValueError): - BayesWorkflowExample(dict(self.init_settings, **{"param_names": None})) + BayesWorkflowExample(**dict(self.init_settings, **{"param_names": None})) # def test_reader_has_read_all_diagnostic_data(self): # assert all(diag_name in self.workflow.reader.keys() From 37fff66438b1d5cf3c825923d6cb4d70c3dc76bd Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 22 Aug 2023 10:59:21 +0100 Subject: [PATCH 161/288] adding mode/run to main call --- indica/writers/bda_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 5a5878a8..73b278c7 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -266,4 +266,4 @@ def query_yes_no( if __name__ == "__main__": - create_nodes(pulse_to_write=23000101) + create_nodes(pulse_to_write=23000101, mode="EDIT", run="RUN02") From d5b5eb51d1ccb310a27f4d7aef9932f952bd2097 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 22 Aug 2023 13:38:08 +0100 Subject: [PATCH 162/288] pressure/stored energy are now saved/written --- indica/bayesmodels.py | 18 ++++++----- indica/workflows/abstract_bayes_workflow.py | 33 ++++++++++++--------- indica/writers/bda_tree.py | 25 +++++++++++----- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index a05e8c77..661e5590 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -8,7 +8,7 @@ warnings.simplefilter("ignore", category=FutureWarning) -PROFILES = [ +PLASMA_ATTRIBUTES = [ "electron_temperature", "electron_density", "ion_temperature", @@ -17,6 +17,10 @@ "fast_density", "neutral_density", "zeff", + "wp", + "wth", + "ptot", + "pth", ] @@ -249,14 +253,14 @@ def ln_posterior(self, parameters: dict, **kwargs): ln_likelihood = self._ln_likelihood() # compare results to data ln_posterior = ln_likelihood + ln_prior - plasma_profiles = {} - for profile_key in PROFILES: - if hasattr(self.plasma, profile_key): - plasma_profiles[profile_key] = getattr(self.plasma, profile_key).sel( + plasma_attributes = {} + for plasma_key in PLASMA_ATTRIBUTES: + if hasattr(self.plasma, plasma_key): + plasma_attributes[plasma_key] = getattr(self.plasma, plasma_key).sel( t=self.plasma.time_to_calculate ) else: - raise ValueError(f"plasma does not have attribute {profile_key}") + raise ValueError(f"plasma does not have attribute {plasma_key}") - blob = deepcopy({**self.bckc, **plasma_profiles}) + blob = deepcopy({**self.bckc, **plasma_attributes}) return ln_posterior, blob diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 4839a2b5..ef3b8fc3 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -140,8 +140,8 @@ def _build_result_dict(self): result = {} quant_list = [item.split(".") for item in self.opt_quantity] - result["TIME"] = self.plasma.t - result["TIME_OPT"] = self.plasma.time_to_calculate + result["TIME_BINS"] = self.plasma.t + result["TIME"] = self.plasma.time_to_calculate result["METADATA"] = { "GITCOMMIT": "PLACEHOLDER", @@ -244,6 +244,8 @@ def _build_result_dict(self): .std(dim="index"), "NFAST_ERR": self.blobs["fast_density"].std(dim="index"), "NNEUTR_ERR": self.blobs["neutral_density"].std(dim="index"), + "ZEFF": self.blobs["zeff"].sum("element").median(dim="index"), + "ZEFF_ERR": self.blobs["zeff"].sum("element").std(dim="index"), } result["PROFILES"] = { **result["PROFILES"], @@ -329,20 +331,23 @@ def _build_result_dict(self): .sel(element=self.plasma.main_ion) .sel(rho_poloidal=0, method="nearest") .median(dim="index"), - "TI0_ERR": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), - "TE0_ERR": self.blobs["electron_temperature"] - .sel(rho_poloidal=0, method="nearest") - .std(dim="index"), - "NE0_ERR": self.blobs["electron_density"] - .sel(rho_poloidal=0, method="nearest") + "WP": self.blobs["wp"] + .median(dim="index"), + "WP_ERR": self.blobs["wp"] .std(dim="index"), - "NI0_ERR": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") + "WTH": self.blobs["wth"] + .median(dim="index"), + "WTH_ERR": self.blobs["wth"] .std(dim="index"), + "PTOT": self.blobs["ptot"] + .median(dim="index"), + "PTOT_ERR": self.blobs["ptot"] + .std(dim="index"), + "PTH": self.blobs["pth"] + .median(dim="index"), + "PTH_ERR": self.blobs["pth"] + .std(dim="index"), + } result["GLOBAL"] = { **result["GLOBAL"], diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 73b278c7..382d79f3 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -9,8 +9,8 @@ def bda(): nodes = { - "TIME": ("NUMERIC", "time vector, s"), - "TIME_OPT": ("NUMERIC", "time of optimisation, s"), + "TIME": ("NUMERIC", "time vector of optimisation, s"), + "TIME_BINS": ("NUMERIC", "time vector used for binning, s"), "INPUT": { "BURN_FRAC": ("NUMERIC", "Burn in fraction for chains"), "ITER": ("NUMERIC", "Iterations of optimiser"), @@ -104,12 +104,21 @@ def bda(): "NI0_ERR": ("SIGNAL", "Central ion density error, m^-3"), "TE0_ERR": ("SIGNAL", "Central electron temperature error, eV"), "TI0_ERR": ("SIGNAL", "Central ion temperature of main ion error, eV"), - "TI0Z1_ERR": ("SIGNAL", "Central ion temperature of impurity Z1, eV"), - "TI0Z2_ERR": ("SIGNAL", "Central ion temperature of impurity Z2, eV"), - "TI0Z3_ERR": ("SIGNAL", "Central ion temperature of impurity Z3, eV"), - "NI0Z1_ERR": ("SIGNAL", "Central density of impurity Z1, m^-3"), - "NI0Z2_ERR": ("SIGNAL", "Central density of impurity Z2, m^-3"), - "NI0Z3_ERR": ("SIGNAL", "Central density of impurity Z3, m^-3"), + "TI0Z1_ERR": ("SIGNAL", "Central ion temperature of impurity Z1 error, eV"), + "TI0Z2_ERR": ("SIGNAL", "Central ion temperature of impurity Z2 error, eV"), + "TI0Z3_ERR": ("SIGNAL", "Central ion temperature of impurity Z3 error, eV"), + "NI0Z1_ERR": ("SIGNAL", "Central density of impurity Z1 error, m^-3"), + "NI0Z2_ERR": ("SIGNAL", "Central density of impurity Z2 error, m^-3"), + "NI0Z3_ERR": ("SIGNAL", "Central density of impurity Z3 error, m^-3"), + "WP": ("SIGNAL", "Stored energy, J"), + "WTH": ("SIGNAL", "Thermal component of stored energy, J"), + "PTOT": ("SIGNAL", "Total pressure, Pa"), + "PTH": ("SIGNAL", "Thermal pressure, Pa"), + "WP_ERR": ("SIGNAL", "Stored energy error, J"), + "WTH_ERR": ("SIGNAL", "Thermal component of stored energy error, J"), + "PTOT_ERR": ("SIGNAL", "Total pressure error, Pa"), + "PTH_ERR": ("SIGNAL", "Thermal pressure error, Pa"), + }, "PHANTOMS": { "FLAG": ("TEXT", "True if phantoms used"), From 07c6895ed6f6902324f88e7d8ed2685100f2ed18 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 22 Aug 2023 14:49:21 +0100 Subject: [PATCH 163/288] removing for loops from pressure/ion density/zeff calculations --- indica/models/plasma.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 0c635f88..789f09ad 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -545,13 +545,7 @@ def pressure_el(self): @property def pressure_th(self): - ion_density = self.ion_density - self._pressure_th.values = self.pressure_el - for elem in self.elements: - self._pressure_th.values += ph.calc_pressure( - ion_density.sel(element=elem).values, - self.ion_temperature.sel(element=elem).values, - ) + self._pressure_th = ph.calc_pressure(self.ion_density, self.ion_temperature).sum("element") + self.pressure_el return self._pressure_th @property @@ -625,14 +619,7 @@ def zeff(self): return self.calc_zeff() # Zeff() def calc_zeff(self): - electron_density = self.electron_density - ion_density = self.ion_density - meanz = self.meanz - for elem in self.elements: - self._zeff.loc[dict(element=elem)] = ( - (ion_density.sel(element=elem) * meanz.sel(element=elem) ** 2) - / electron_density - ).values + self._zeff = (self.ion_density * self.meanz) ** 2 / self.electron_density return self._zeff @property @@ -753,6 +740,7 @@ def calc_sxr_radiation(self): @property def meanz(self): fz = self.fz + for elem in self.elements: self._meanz.loc[dict(element=elem)] = ( (fz[elem] * fz[elem].ion_charges).sum("ion_charges").values From 845954b7073a7b4cd183fafa0ca8a822852e560c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 22 Aug 2023 15:12:14 +0100 Subject: [PATCH 164/288] removing for loops from ion density calculations --- indica/models/plasma.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 789f09ad..8708ac9b 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -628,18 +628,15 @@ def ion_density(self): def calc_ion_density(self): meanz = self.meanz - main_ion_density = self.electron_density - self.fast_density * meanz.sel( - element=self.main_ion - ) for elem in self.impurities: self._ion_density.loc[dict(element=elem)] = self.impurity_density.sel( element=elem ).values - main_ion_density -= self.impurity_density.sel(element=elem) * meanz.sel( - element=elem - ) - self._ion_density.loc[dict(element=self.main_ion)] = main_ion_density.values + main_ion_density = self.electron_density - self.fast_density * meanz.sel(element=self.main_ion) \ + - (self.impurity_density * meanz).sum("element") + + self._ion_density.loc[dict(element=self.main_ion)] = main_ion_density return self._ion_density @property From 83ede9e082063282131dfb2a0351a3a7e49a8b11 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 22 Aug 2023 16:00:48 +0100 Subject: [PATCH 165/288] defaulting to using cached calculations -> factor 2 speed upgit add indica/models/plasma.py --- indica/models/plasma.py | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 8708ac9b..b119af32 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -419,6 +419,28 @@ def initialize_variables(self): ], ) + self.Pth = CachedCalculation( + self.calc_pth, + [ + self.electron_density, + self.ion_density, + self.electron_temperature, + self.ion_temperature, + ], + ) + + self.Ptot = CachedCalculation( + self.calc_ptot, + [ + self.electron_density, + self.ion_density, + self.electron_temperature, + self.ion_temperature, + self.pressure_fast, + ], + ) + + self.Lz_tot = CachedCalculation( self.calc_lz_tot, [ @@ -563,6 +585,9 @@ def pressure_fast(self): @property def pth(self): + return self.Pth() + + def calc_pth(self): pressure_th = self.pressure_th for t in np.array(self.time_to_calculate, ndmin=1): self._pth.loc[dict(t=t)] = np.trapz( @@ -572,6 +597,9 @@ def pth(self): @property def ptot(self): + return self.Ptot() + + def calc_ptot(self): pressure_tot = self.pressure_tot for t in np.array(self.time_to_calculate, ndmin=1): self._ptot.loc[dict(t=t)] = np.trapz( @@ -593,7 +621,8 @@ def wp(self): @property def fz(self): - return self.calc_fz() # self.Fz() + return self.Fz() + # return self.calc_fz() # self.Fz() def calc_fz(self): for elem in self.elements: @@ -616,7 +645,8 @@ def calc_fz(self): @property def zeff(self): - return self.calc_zeff() # Zeff() + return self.Zeff() + # return self.calc_zeff() def calc_zeff(self): self._zeff = (self.ion_density * self.meanz) ** 2 / self.electron_density @@ -624,7 +654,8 @@ def calc_zeff(self): @property def ion_density(self): - return self.calc_ion_density() # self.Ion_density() + return self.Ion_density() + # return self.calc_ion_density() def calc_ion_density(self): meanz = self.meanz @@ -1219,7 +1250,6 @@ def __init__(self, operator: Callable, dependencies: list, verbose: bool = False @lru_cache() def __call__(self): - print("Recalculating") if self.verbose: print("Recalculating") return self.operator() From ca5900cc4850b74294e3ab197feb5d696f62b4e9 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 22 Aug 2023 16:25:51 +0100 Subject: [PATCH 166/288] example transform moved to own function --- indica/models/thomson_scattering.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/indica/models/thomson_scattering.py b/indica/models/thomson_scattering.py index d74498ee..d7bcb885 100644 --- a/indica/models/thomson_scattering.py +++ b/indica/models/thomson_scattering.py @@ -111,6 +111,18 @@ def __call__( return self.bckc +def ts_transform_example(nchannels): + x_positions = np.linspace(0.2, 0.8, nchannels) + y_positions = np.linspace(0.0, 0.0, nchannels) + z_positions = np.linspace(0.0, 0.0, nchannels) + transform = TransectCoordinates( + x_positions, + y_positions, + z_positions, + "ts", + machine_dimensions=((0.15, 0.95), (-0.7, 0.7)), + ) + return transform def example_run( pulse: int = None, @@ -124,19 +136,7 @@ def example_run( if plasma is None: plasma = example_plasma(pulse=pulse) - # Create new interferometers diagnostics - nchannels = 11 - x_positions = np.linspace(0.2, 0.8, nchannels) - y_positions = np.linspace(0.0, 0.0, nchannels) - z_positions = np.linspace(0.0, 0.0, nchannels) - - transect_transform = TransectCoordinates( - x_positions, - y_positions, - z_positions, - diagnostic_name, - machine_dimensions=plasma.machine_dimensions, - ) + transect_transform = ts_transform_example(11) transect_transform.set_equilibrium(plasma.equilibrium) model = ThomsonScattering( diagnostic_name, From 741808e408f75fddb030eaf9e151d668875c3397 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 22 Aug 2023 17:09:41 +0100 Subject: [PATCH 167/288] t = time_to_calculate and bckc don't print if chi2 not included --- indica/models/thomson_scattering.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/indica/models/thomson_scattering.py b/indica/models/thomson_scattering.py index d7bcb885..94af48e8 100644 --- a/indica/models/thomson_scattering.py +++ b/indica/models/thomson_scattering.py @@ -41,6 +41,9 @@ def _build_bckc_dictionary(self): self.bckc[quantity] = self.Te_at_channels long_name = "Te" units = "eV" + elif quant == "chi2": + # Placeholder + continue else: print(f"{quant} not available in model for {self.instrument_method}") continue @@ -82,7 +85,7 @@ def __call__( """ if self.plasma is not None: if t is None: - t = self.plasma.t + t = self.plasma.time_to_calculate Ne = self.plasma.electron_density.interp(t=t) Te = self.plasma.electron_temperature.interp(t=t) else: From cf3b01f9a0de8c62af581f250a497f358b499efd Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 23 Aug 2023 10:49:41 +0100 Subject: [PATCH 168/288] Thompson Scattering added --- indica/workflows/bayes_plots.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 93955202..2f3f8cdd 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -347,6 +347,28 @@ def plot_bayes_result( ylabel="Temperature [eV]", xlabel="Channel", ) + key = "TS.TE" + if key in model_data.keys(): + _plot_1d( + model_data, + key, + diag_data, + f"{key.replace('.', '_')}" + filetype, + figheader=figheader, + ylabel="Temperature [eV]", + xlabel="Channel", + ) + key = "TS.NE" + if key in model_data.keys(): + _plot_1d( + model_data, + key, + diag_data, + f"{key.replace('.', '_')}" + filetype, + figheader=figheader, + ylabel="Density [m^-3]", + xlabel="Channel", + ) key = "TE" plot_profile( From e4fe2bcc0f534dc68be014d6d88cd499d361263e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 23 Aug 2023 13:53:55 +0100 Subject: [PATCH 169/288] error checking for missing data --- indica/workflows/abstract_bayes_workflow.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index ef3b8fc3..cce32692 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -47,6 +47,11 @@ def read_test_data( def read_data(self, diagnostics: list, tstart=None, tend=None, dt=None): self.reader = ReadST40(self.pulse, tstart=tstart, tend=tend, dt=dt) self.reader(diagnostics) + + missing_keys = set(diagnostics) - set(self.reader.binned_data.keys()) + if len(missing_keys) > 0: + raise ValueError(f"missing data: {missing_keys}") + self.equilibrium = self.reader.equilibrium self.transforms = self.reader.transforms self.data = self.reader.binned_data From b31616353ba690a468992bd5f67db1aa9cf2371f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 24 Aug 2023 09:55:32 +0100 Subject: [PATCH 170/288] in _make_spectra moved sorting wavelengths to end of method and fixed wavelength co-ordinates that were added incorrectly --- indica/models/helike_spectroscopy.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 90ac5c4e..3e0c0673 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -204,8 +204,6 @@ def _make_spectra(self, calc_rho: bool = False): ), ) spectra = xr.concat([_spectra, empty], "wavelength") - spectra = spectra.sortby("wavelength") - self.spectra = spectra measured_spectra = self.los_transform.integrate_on_los( @@ -213,9 +211,10 @@ def _make_spectra(self, calc_rho: bool = False): t=self.spectra.t, calc_rho=calc_rho, ) - self.measured_spectra = measured_spectra.assign_coords( - {"wavelength": self.window.wavelength} + measured_spectra = measured_spectra.assign_coords( + {"wavelength": self.spectra.wavelength} ) + self.measured_spectra = measured_spectra.sortby("wavelength") self.spectra_los = self.los_transform.along_los def _moment_analysis(self): From bc56b39feffa4400d8377bfdf39c848baa789353 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 24 Aug 2023 10:47:20 +0100 Subject: [PATCH 171/288] integrating spectra was removing nans so added them back post integration --- indica/models/helike_spectroscopy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 3e0c0673..5d78e0c9 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -214,6 +214,7 @@ def _make_spectra(self, calc_rho: bool = False): measured_spectra = measured_spectra.assign_coords( {"wavelength": self.spectra.wavelength} ) + measured_spectra[measured_spectra==0] = np.nan self.measured_spectra = measured_spectra.sortby("wavelength") self.spectra_los = self.los_transform.along_los From 5368990323b814ed4567ca6c85a6c38b1057f37d Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 24 Aug 2023 13:45:23 +0100 Subject: [PATCH 172/288] pixel offset added as __call__ option --- indica/models/helike_spectroscopy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 5d78e0c9..ac6e9dd5 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -386,6 +386,7 @@ def __call__( calc_rho: bool = False, moment_analysis: bool = False, background: int = None, + pixel_offset: int = None, **kwargs, ): """ @@ -462,6 +463,9 @@ def __call__( if background is not None: self.measured_spectra = self.measured_spectra + background + if pixel_offset is not None: + self.measured_spectra = self.measured_spectra.shift(wavelength=round(pixel_offset), fill_value=np.nan) + self._build_bckc_dictionary() return self.bckc From 82208cae7631cfdc1efdd4fab46610d6bc20ccfb Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 24 Aug 2023 14:33:33 +0100 Subject: [PATCH 173/288] time vector in INPUTS added --- indica/workflows/abstract_bayes_workflow.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index cce32692..23e35b05 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -162,9 +162,13 @@ def _build_result_dict(self): "OPT_QUANTITY": self.opt_quantity, "PARAM_NAMES": self.param_names, "PULSE": self.pulse, - "DT": self.dt, "IMPURITIES": self.plasma.impurities, "MAIN_ION": self.plasma.main_ion, + "TSTART":self.tstart, + "TEND": self.tend, + "DT": self.dt, + "TSAMPLE": self.tsample, + } result["INPUT"]["WORKFLOW"] = { diag_name.upper(): { From da94202dded0a073799e40e026ef68d1be5c6d7b Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 4 Sep 2023 10:16:41 +0100 Subject: [PATCH 174/288] adjusting calibration factor --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index ac6e9dd5..dcf50731 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -36,7 +36,7 @@ def __init__( name: str, instrument_method="get_helike_spectroscopy", etendue: float = 1.0, - calibration: float = 8.0e-20, + calibration: float = 1.0e-18, element: str = "ar", window_len: int = 1030, window_lim=None, From 0abbaaa411d271add4e73fe5031a74a078520603 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 4 Sep 2023 10:17:57 +0100 Subject: [PATCH 175/288] max-min ranges now 0.5-99.5 --- indica/workflows/bayes_plots.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 2f3f8cdd..127c0cdd 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -45,8 +45,8 @@ def plot_profile( ) plt.fill_between( profile.rho_poloidal, - profile.quantile(0.00, dim="index"), - profile.quantile(1.00, dim="index"), + profile.quantile(0.005, dim="index"), + profile.quantile(0.995, dim="index"), label=f"{blobkey}, Max-Min", zorder=1, color="lightgrey", @@ -118,8 +118,8 @@ def _plot_1d( ) plt.fill_between( blob_data.__getattr__(dims[0]), - blob_data.quantile(0.00, dim="index"), - blob_data.quantile(1.00, dim="index"), + blob_data.quantile(0.005, dim="index"), + blob_data.quantile(0.995, dim="index"), label=f"{blobkey}, Max-Min", zorder=1, color="lightgrey", @@ -263,12 +263,19 @@ def plot_bayes_result( plot_autocorr(autocorr, param_names, figheader, filetype=filetype) if "CXFF_PI.TI" in model_data.keys(): + max_channel = diag_data["CXFF_PI.TI"].sel(t=model_data["CXFF_PI.TI"].t).idxmax("channel").values model_data["CXFF_PI.TI0"] = model_data["CXFF_PI.TI"].sel( - channel=diag_data["CXFF_PI.TI"].channel - ) + channel=max_channel + ) diag_data["CXFF_PI.TI0"] = diag_data["CXFF_PI.TI"].sel( - channel=diag_data["CXFF_PI.TI"].channel + channel=max_channel ) + # model_data["CXFF_PI.TI0"] = model_data["CXFF_PI.TI"].sel( + # channel=diag_data["CXFF_PI.TI"].channel + # ) + # diag_data["CXFF_PI.TI0"] = diag_data["CXFF_PI.TI"].sel( + # channel=diag_data["CXFF_PI.TI"].channel + # ) key = "CXFF_PI.TI0" if key in model_data.keys(): From 6114dc4e3ad1f9f096d2c2a91092ee22179540c8 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 4 Sep 2023 10:18:40 +0100 Subject: [PATCH 176/288] high_density_sampling now uses 3 best points --- indica/bayesmodels.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 661e5590..102cdd65 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -200,10 +200,11 @@ def sample_from_priors(self, param_names, size=10): def sample_from_high_density_region( self, param_names: list, sampler, nwalkers: int, nsamples=100 ): + # TODO: implement smarter MLE (maximum likelihood estimate) start_points = self.sample_from_priors(param_names, size=nsamples) ln_prob, _ = sampler.compute_log_prob(start_points) - num_best_points = int(nsamples * 0.05) + num_best_points = 3 index_best_start = np.argsort(ln_prob)[-num_best_points:] best_start_points = start_points[index_best_start, :] best_points_std = np.std(best_start_points, axis=0) From d210b2886e80eedecbfc299d9b64dd65bb604eee Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 4 Sep 2023 10:19:20 +0100 Subject: [PATCH 177/288] Gelmin-Rubin diagnostic added to optimisation node --- indica/writers/bda_tree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 382d79f3..d5c7c635 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -142,6 +142,7 @@ def bda(): "AUTO_CORR": ("NUMERIC", "Auto-correlation (iteration nwalker)"), "POST_SAMPLE": ("NUMERIC", "Posterior probability samples (sample)"), "PRIOR_SAMPLE": ("NUMERIC", "Prior samples"), + "GELMANRUBIN": ("NUMERIC", "Gelmin-Rubin convergence diagnostic"), }, } return nodes From 7e7d02dae3bfba0f2807e853f421675ec20f691e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 4 Sep 2023 11:27:03 +0100 Subject: [PATCH 178/288] Gelman-Rubin diagnostic added to tree --- indica/workflows/abstract_bayes_workflow.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 23e35b05..133d8b8c 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -323,6 +323,7 @@ def _build_result_dict(self): "PRIOR_SAMPLE": self.prior_sample, "POST_SAMPLE": self.post_sample, "AUTOCORR": self.autocorr, + "GELMANRUBIN": gelman_rubin(self.sampler.get_chain(flat=False)) } result["GLOBAL"] = { @@ -438,7 +439,7 @@ def run_sampler(self, iterations, burn_frac): self.prior_sample = self.bayesmodel.sample_from_priors( self.param_names, size=int(1e4) ) - self.post_sample = self.sampler.get_chain(flat=True) + self.post_sample = self.sampler.get_chain(discard=int(iterations * burn_frac), flat=True) self.result = self._build_result_dict() def save_pickle(self, filepath): @@ -491,3 +492,16 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl : sampler.iteration, ] return autocorr + + +def gelman_rubin(chain): + ssq = np.var(chain, axis=1, ddof=1) + w = np.mean(ssq, axis=0) + theta_b = np.mean(chain, axis=1) + theta_bb = np.mean(theta_b, axis=0) + m = chain.shape[0] + n = chain.shape[1] + B = n/(m-1) * np.sum((theta_bb - theta_b)**2, axis=0) + var_theta = (n-1) / n * w + 1 / n * B + R = np.sqrt(var_theta / w) + return R \ No newline at end of file From e3c2abf2f3e9f2e794fcf8fb8c643e0ade66adf4 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 4 Sep 2023 11:27:39 +0100 Subject: [PATCH 179/288] stashing --- indica/workflows/bayes_workflow_example.py | 94 ++++++++++++++++------ 1 file changed, 68 insertions(+), 26 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index cb81a09d..e1e72f5e 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -15,6 +15,8 @@ from indica.models.helike_spectroscopy import HelikeSpectrometer from indica.models.interferometry import Interferometry from indica.models.interferometry import smmh1_transform_example +from indica.models.thomson_scattering import ThomsonScattering +from indica.models.thomson_scattering import ts_transform_example from indica.models.plasma import Plasma from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow from indica.workflows.bayes_plots import plot_bayes_result @@ -32,8 +34,8 @@ "Nimp_prof.y0": 1e17, "Nimp_prof.y1": 5e15, "Nimp_prof.wcenter": 0.4, - "Nimp_prof.wped": 6, - "Nimp_prof.peaking": 2, + "Nimp_prof.wped": 2, + "Nimp_prof.peaking": 4, "Te_prof.y0": 3000, "Te_prof.y1": 50, "Te_prof.wcenter": 0.4, @@ -47,16 +49,16 @@ } DEFAULT_PRIORS = { - "Ne_prof.y0": get_uniform(2e19, 4e20), - "Ne_prof.y1": get_uniform(1e18, 1e19), + "Ne_prof.y0": get_uniform(1e19, 2e20), + "Ne_prof.y1": get_uniform(1e18, 2e19), "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), "Ne_prof.wped": get_uniform(1, 6), "Ne_prof.wcenter": get_uniform(0.1, 0.8), - "Ne_prof.peaking": get_uniform(1, 6), + "Ne_prof.peaking": get_uniform(1, 10), "Nimp_prof.y0": loguniform(1e16, 1e18), - "Nimp_prof.y1": get_uniform(1e15, 1e16), + "Nimp_prof.y1": loguniform(1e15, 1e17), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( - (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 + (x1 > x2 * 10) & (x1 < x2 * 1e5), 1, 0 ), "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), "Nimp_prof.wped": get_uniform(1, 6), @@ -74,21 +76,22 @@ "Ti_prof.wped": get_uniform(1, 6), "Ti_prof.wcenter": get_uniform(0.1, 0.8), "Ti_prof.peaking": get_uniform(1, 10), + "xrcs.pixel_offset": get_uniform(-4.01, -3.99), } OPTIMISED_PARAMS = [ # "Ne_prof.y1", "Ne_prof.y0", - # "Ne_prof.peaking", - # "Ne_prof.wcenter", - # "Ne_prof.wped", + "Ne_prof.peaking", + "Ne_prof.wcenter", + "Ne_prof.wped", # "Nimp_prof.y1", "Nimp_prof.y0", # "Nimp_prof.wcenter", # "Nimp_prof.wped", - # "Nimp_prof.peaking", + "Nimp_prof.peaking", "Te_prof.y0", - # "Te_prof.wped", + "Te_prof.wped", "Te_prof.wcenter", "Te_prof.peaking", "Ti_prof.y0", @@ -98,9 +101,11 @@ ] OPTIMISED_QUANTITY = [ "xrcs.spectra", - "cxff_pi.ti", + # "cxff_pi.ti", "efit.wp", - "smmh1.ne", + # "smmh1.ne", + "ts.te", + "ts.ne", ] @@ -128,6 +133,7 @@ def __init__( self.tstart = tstart self.tend = tend self.dt = dt + self.model_kwargs = {} for attribute in [ "param_names", @@ -151,6 +157,7 @@ def __init__( "xrcs": helike_transform_example(1), "smmh1": smmh1_transform_example(1), "cxff_pi": pi_transform_example(5), + "ts":ts_transform_example(11), } self.read_test_data( example_transforms, tstart=self.tstart, tend=self.tend, dt=self.dt @@ -246,6 +253,13 @@ def setup_models(self, diagnostics: list): transform.set_equilibrium(self.equilibrium) model = ChargeExchange(name=diag, element="ar") model.set_transect_transform(transform) + + elif diag == "ts": + transform = self.transforms[diag] + transform.set_equilibrium(self.equilibrium) + model = ThomsonScattering(name=diag, ) + model.set_transect_transform(transform) + else: raise ValueError(f"{diag} not found in setup_models") self.models[diag] = model @@ -283,6 +297,17 @@ def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2, drop=True) + + if "ts" in self.diagnostics: + _ts_data = self.models["ts"]() + ts_data = {key: _ts_data[key].expand_dims(dim={"t": [self.plasma.time_to_calculate]}) for key in ["te", "ne"]} + opt_data["ts.te"] = ts_data["te"] + opt_data["ts.ne"] = ts_data["ne"] + opt_data["ts.te"]["error"] = opt_data["ts.te"] / opt_data["ts.te"] * ( + 0.10 * opt_data["ts.te"].max(dim="channel")) + opt_data["ts.ne"]["error"] = opt_data["ts.ne"] / opt_data["ts.ne"] * ( + 0.10 * opt_data["ts.ne"].max(dim="channel")) + if "efit" in self.diagnostics: opt_data["efit.wp"] = ( self.models["efit"]() @@ -291,6 +316,7 @@ def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): ) if noise: + #TODO: add TS opt_data["smmh1.ne"] = opt_data["smmh1.ne"] + opt_data[ "smmh1.ne" ].max().values * np.random.normal(0, noise_factor, None) @@ -317,14 +343,14 @@ def _exp_data(self, **kwargs): opt_data = flatdict.FlatDict(self.data, ".") if "xrcs" in self.diagnostics: opt_data["xrcs.spectra"]["wavelength"] = ( - opt_data["xrcs.spectra"].wavelength * 0.1 + opt_data["xrcs.spectra"].wavelength ) background = opt_data["xrcs.spectra"].where( (opt_data["xrcs.spectra"].wavelength < 0.392) & (opt_data["xrcs.spectra"].wavelength > 0.388), drop=True, ) - self.model_kwargs["xrcs_background"] = background.mean( + self.model_kwargs["xrcs.background"] = background.mean( dim="wavelength" ).sel(t=self.plasma.time_to_calculate) opt_data["xrcs.spectra"]["error"] = np.sqrt( @@ -333,8 +359,16 @@ def _exp_data(self, **kwargs): if "cxff_pi" in self.diagnostics: opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( - opt_data["cxff_pi"]["ti"].channel == 2, drop=True + opt_data["cxff_pi"]["ti"] != 0, ) + if "ts" in self.diagnostics: + # TODO: fix error, for now flat error + opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel >19, drop=True) + opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel >19, drop=True) + + opt_data["ts.te"]["error"] = opt_data["ts.te"] * 0.10 + 10 + opt_data["ts.ne"]["error"] = opt_data["ts.ne"] * 0.10 + 10 + return opt_data def setup_optimiser( @@ -343,6 +377,7 @@ def setup_optimiser( nwalkers=50, burn_frac=0.10, sample_high_density=False, + **kwargs, ): self.model_kwargs = model_kwargs self.nwalkers = nwalkers @@ -368,13 +403,13 @@ def setup_optimiser( kwargs=model_kwargs, ) self.start_points = self._sample_start_points( - sample_high_density=self.sample_high_density + sample_high_density=self.sample_high_density, **kwargs ) - def _sample_start_points(self, sample_high_density: bool = True): + def _sample_start_points(self, sample_high_density: bool = True, nsamples=100, **kwargs): if sample_high_density: start_points = self.bayesmodel.sample_from_high_density_region( - self.param_names, self.sampler, self.nwalkers, nsamples=100 + self.param_names, self.sampler, self.nwalkers, nsamples=nsamples ) else: start_points = self.bayesmodel.sample_from_priors( @@ -399,7 +434,7 @@ def __call__( pulse_to_write=pulse_to_write, run=run, diagnostic_quantities=self.opt_quantity, - mode="NEW", + mode="EDIT", ) self.run_sampler(iterations=iterations, burn_frac=burn_frac) @@ -418,7 +453,13 @@ def __call__( if __name__ == "__main__": run = BayesWorkflowExample( pulse=None, - diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], + diagnostics=[ + "xrcs", + "efit", + "smmh1", + "cxff_pi", + "ts", + ], param_names=OPTIMISED_PARAMS, opt_quantity=OPTIMISED_QUANTITY, priors=DEFAULT_PRIORS, @@ -430,16 +471,17 @@ def __call__( ) run.setup_plasma( - tsample=0.060, + tsample=0.05, ) run.setup_opt_data(phantoms=run.phantoms) - run.setup_optimiser(nwalkers=50, sample_high_density=True, model_kwargs={}) + run.setup_optimiser(nwalkers=50, sample_high_density=True, model_kwargs=run.model_kwargs) results = run( - filepath="./results/test/", - pulse_to_write=23000101, + filepath=f"./results/test/", + pulse_to_write=25000000, run="RUN01", mds_write=True, plot=True, burn_frac=0.10, iterations=100, ) + From cb0df2ab4998fcf0656156ed66ecaa78e4cf8499 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 6 Sep 2023 16:53:03 +0100 Subject: [PATCH 180/288] now accepts variable number of input parameters and fills missing values with default settings --- indica/profiles_gauss.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/indica/profiles_gauss.py b/indica/profiles_gauss.py index e266a4ed..67b1cf7e 100644 --- a/indica/profiles_gauss.py +++ b/indica/profiles_gauss.py @@ -32,6 +32,7 @@ def __init__( self.xend = xend self.coord = f"rho_{coord}" self.x = np.linspace(0, 1, 15) ** 0.7 + # self.x = 1-np.logspace(0, -2, 30) self.datatype = datatype if xspl is None: xspl = np.linspace(0, 1.0, 30) @@ -55,6 +56,9 @@ def __init__( if parameters is None: parameters = get_defaults(datatype) + elif {"y0", "y1", "yend", "wcenter", "wped", "peaking",} >= set(parameters): + _parameters = get_defaults(datatype) + parameters = dict(_parameters, **parameters) for k, p in parameters.items(): setattr(self, k, p) From b48ea95ae40eec2d0a99102e975948343715b124 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 6 Sep 2023 16:57:44 +0100 Subject: [PATCH 181/288] priors now more constrained for wped/wcenter --- indica/workflows/bayes_workflow_example.py | 64 ++++++++++++---------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index cb81a09d..6e000364 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -24,75 +24,83 @@ # global configurations DEFAULT_PROFILE_PARAMS = { "Ne_prof.y0": 5e19, - "Ne_prof.wcenter": 0.4, - "Ne_prof.peaking": 2, "Ne_prof.y1": 2e18, "Ne_prof.yend": 1e18, - "Ne_prof.wped": 2, + "Ne_prof.wped": 3, + "Ne_prof.wcenter": 0.3, + "Ne_prof.peaking": 1.2, + "Nimp_prof.y0": 1e17, "Nimp_prof.y1": 5e15, - "Nimp_prof.wcenter": 0.4, + "Nimp_prof.yend": 1e15, + "Nimp_prof.wcenter": 0.3, "Nimp_prof.wped": 6, "Nimp_prof.peaking": 2, + "Te_prof.y0": 3000, "Te_prof.y1": 50, - "Te_prof.wcenter": 0.4, + "Te_prof.yend": 10, + "Te_prof.wcenter": 0.2, "Te_prof.wped": 3, - "Te_prof.peaking": 2, + "Te_prof.peaking": 1.5, + "Ti_prof.y0": 6000, "Ti_prof.y1": 50, - "Ti_prof.wcenter": 0.4, + "Ti_prof.yend": 10, + "Ti_prof.wcenter": 0.2, "Ti_prof.wped": 3, - "Ti_prof.peaking": 2, + "Ti_prof.peaking": 1.5, } DEFAULT_PRIORS = { "Ne_prof.y0": get_uniform(2e19, 4e20), "Ne_prof.y1": get_uniform(1e18, 1e19), "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), - "Ne_prof.wped": get_uniform(1, 6), - "Ne_prof.wcenter": get_uniform(0.1, 0.8), - "Ne_prof.peaking": get_uniform(1, 6), + "Ne_prof.wped": get_uniform(2, 6), + "Ne_prof.wcenter": get_uniform(0.2, 0.4), + "Ne_prof.peaking": get_uniform(1, 4), + "Nimp_prof.y0": loguniform(1e16, 1e18), "Nimp_prof.y1": get_uniform(1e15, 1e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 ), "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), - "Nimp_prof.wped": get_uniform(1, 6), - "Nimp_prof.wcenter": get_uniform(0.1, 0.8), - "Nimp_prof.peaking": get_uniform(1, 10), + "Nimp_prof.wped": get_uniform(2, 6), + "Nimp_prof.wcenter": get_uniform(0.2, 0.4), + "Nimp_prof.peaking": get_uniform(1, 6), "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where( (x1 > x2), 1, 0 ), # impurity always more peaked + "Te_prof.y0": get_uniform(1000, 5000), - "Te_prof.wped": get_uniform(1, 6), - "Te_prof.wcenter": get_uniform(0.1, 0.8), - "Te_prof.peaking": get_uniform(1, 10), + "Te_prof.wped": get_uniform(2, 6), + "Te_prof.wcenter": get_uniform(0.2, 0.4), + "Te_prof.peaking": get_uniform(1, 4), # "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode "Ti_prof.y0": get_uniform(1000, 10000), - "Ti_prof.wped": get_uniform(1, 6), - "Ti_prof.wcenter": get_uniform(0.1, 0.8), - "Ti_prof.peaking": get_uniform(1, 10), + "Ti_prof.wped": get_uniform(2, 6), + "Ti_prof.wcenter": get_uniform(0.2, 0.4), + "Ti_prof.peaking": get_uniform(1, 6), } OPTIMISED_PARAMS = [ - # "Ne_prof.y1", + "Ne_prof.y1", "Ne_prof.y0", - # "Ne_prof.peaking", - # "Ne_prof.wcenter", - # "Ne_prof.wped", + "Ne_prof.peaking", + "Ne_prof.wcenter", + "Ne_prof.wped", # "Nimp_prof.y1", "Nimp_prof.y0", # "Nimp_prof.wcenter", # "Nimp_prof.wped", - # "Nimp_prof.peaking", + "Nimp_prof.peaking", "Te_prof.y0", - # "Te_prof.wped", + "Te_prof.wped", "Te_prof.wcenter", "Te_prof.peaking", "Ti_prof.y0", - # "Ti_prof.wped", + "Ti_prof.wped", "Ti_prof.wcenter", "Ti_prof.peaking", ] @@ -374,7 +382,7 @@ def setup_optimiser( def _sample_start_points(self, sample_high_density: bool = True): if sample_high_density: start_points = self.bayesmodel.sample_from_high_density_region( - self.param_names, self.sampler, self.nwalkers, nsamples=100 + self.param_names, self.sampler, self.nwalkers, nsamples=200 ) else: start_points = self.bayesmodel.sample_from_priors( From 90c38fb8d75e144293586cf161d647af7a7f3bf4 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 6 Sep 2023 17:06:25 +0100 Subject: [PATCH 182/288] priors now more constrained for wped/wcenter --- indica/workflows/bayes_workflow_example.py | 66 +++++++++++++--------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index e1e72f5e..bbfa1b75 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -23,64 +23,74 @@ from indica.writers.bda_tree import create_nodes from indica.writers.bda_tree import write_nodes + + # global configurations DEFAULT_PROFILE_PARAMS = { "Ne_prof.y0": 5e19, - "Ne_prof.wcenter": 0.4, - "Ne_prof.peaking": 2, "Ne_prof.y1": 2e18, "Ne_prof.yend": 1e18, - "Ne_prof.wped": 2, + "Ne_prof.wped": 3, + "Ne_prof.wcenter": 0.3, + "Ne_prof.peaking": 1.2, + "Nimp_prof.y0": 1e17, "Nimp_prof.y1": 5e15, - "Nimp_prof.wcenter": 0.4, - "Nimp_prof.wped": 2, - "Nimp_prof.peaking": 4, + "Nimp_prof.yend": 1e15, + "Nimp_prof.wcenter": 0.3, + "Nimp_prof.wped": 6, + "Nimp_prof.peaking": 2, + "Te_prof.y0": 3000, "Te_prof.y1": 50, - "Te_prof.wcenter": 0.4, + "Te_prof.yend": 10, + "Te_prof.wcenter": 0.2, "Te_prof.wped": 3, - "Te_prof.peaking": 2, + "Te_prof.peaking": 1.5, + "Ti_prof.y0": 6000, "Ti_prof.y1": 50, - "Ti_prof.wcenter": 0.4, + "Ti_prof.yend": 10, + "Ti_prof.wcenter": 0.2, "Ti_prof.wped": 3, - "Ti_prof.peaking": 2, + "Ti_prof.peaking": 1.5, } DEFAULT_PRIORS = { - "Ne_prof.y0": get_uniform(1e19, 2e20), - "Ne_prof.y1": get_uniform(1e18, 2e19), + "Ne_prof.y0": get_uniform(2e19, 4e20), + "Ne_prof.y1": get_uniform(1e18, 1e19), "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), - "Ne_prof.wped": get_uniform(1, 6), - "Ne_prof.wcenter": get_uniform(0.1, 0.8), - "Ne_prof.peaking": get_uniform(1, 10), + "Ne_prof.wped": get_uniform(2, 6), + "Ne_prof.wcenter": get_uniform(0.2, 0.4), + "Ne_prof.peaking": get_uniform(1, 4), + "Nimp_prof.y0": loguniform(1e16, 1e18), - "Nimp_prof.y1": loguniform(1e15, 1e17), + "Nimp_prof.y1": get_uniform(1e15, 1e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( - (x1 > x2 * 10) & (x1 < x2 * 1e5), 1, 0 + (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 ), "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), - "Nimp_prof.wped": get_uniform(1, 6), - "Nimp_prof.wcenter": get_uniform(0.1, 0.8), - "Nimp_prof.peaking": get_uniform(1, 10), + "Nimp_prof.wped": get_uniform(2, 6), + "Nimp_prof.wcenter": get_uniform(0.2, 0.4), + "Nimp_prof.peaking": get_uniform(1, 6), "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where( (x1 > x2), 1, 0 ), # impurity always more peaked + "Te_prof.y0": get_uniform(1000, 5000), - "Te_prof.wped": get_uniform(1, 6), - "Te_prof.wcenter": get_uniform(0.1, 0.8), - "Te_prof.peaking": get_uniform(1, 10), + "Te_prof.wped": get_uniform(2, 6), + "Te_prof.wcenter": get_uniform(0.2, 0.4), + "Te_prof.peaking": get_uniform(1, 4), # "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode "Ti_prof.y0": get_uniform(1000, 10000), - "Ti_prof.wped": get_uniform(1, 6), - "Ti_prof.wcenter": get_uniform(0.1, 0.8), - "Ti_prof.peaking": get_uniform(1, 10), + "Ti_prof.wped": get_uniform(2, 6), + "Ti_prof.wcenter": get_uniform(0.2, 0.4), + "Ti_prof.peaking": get_uniform(1, 6), "xrcs.pixel_offset": get_uniform(-4.01, -3.99), } OPTIMISED_PARAMS = [ - # "Ne_prof.y1", + "Ne_prof.y1", "Ne_prof.y0", "Ne_prof.peaking", "Ne_prof.wcenter", @@ -95,7 +105,7 @@ "Te_prof.wcenter", "Te_prof.peaking", "Ti_prof.y0", - # "Ti_prof.wped", + "Ti_prof.wped", "Ti_prof.wcenter", "Ti_prof.peaking", ] From 51ee0d9ca803ab6fd8c221c39be0fb8ae8eff1d5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 11 Sep 2023 13:28:11 +0100 Subject: [PATCH 183/288] fast particles from ASTRA included --- indica/workflows/bayes_workflow_example.py | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index bbfa1b75..9b5651a6 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -22,7 +22,7 @@ from indica.workflows.bayes_plots import plot_bayes_result from indica.writers.bda_tree import create_nodes from indica.writers.bda_tree import write_nodes - +from indica.readers.read_st40 import ReadST40 # global configurations @@ -129,6 +129,7 @@ def __init__( profile_params: dict, pulse: int = None, phantoms: bool = False, + fast_particles = False, tstart=0.02, tend=0.10, dt=0.005, @@ -140,6 +141,7 @@ def __init__( self.priors = priors self.profile_params = profile_params self.phantoms = phantoms + self.fast_particles = fast_particles self.tstart = tstart self.tend = tend self.dt = dt @@ -212,6 +214,9 @@ def setup_plasma( ] self.plasma.set_equilibrium(self.equilibrium) self.plasma.update_profiles(self.profile_params) + if self.fast_particles: + self._init_fast_particles() + self.plasma.build_atomic_data(calc_power_loss=False) self.save_phantom_profiles() @@ -274,6 +279,26 @@ def setup_models(self, diagnostics: list): raise ValueError(f"{diag} not found in setup_models") self.models[diag] = model + def _init_fast_particles(self): + st40_code = ReadST40(13000000 + self.pulse, self.tstart, self.tend, dt=self.dt, tree="astra") + st40_code.get_raw_data("", "astra", "RUN602") + st40_code.bin_data_in_time(["astra"], self.tstart, self.tend, self.dt) + code_data = st40_code.binned_data["astra"] + Nf = ( + code_data["nf"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + * 1.0e19 + ) + self.plasma.fast_density.values = Nf.values + Nn = ( + code_data["nn"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + * 1.0e19 + ) + self.plasma.neutral_density.values = Nn.values + Pblon = code_data["pblon"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + self.plasma.pressure_fast_parallel.values = Pblon.values + Pbper = code_data["pbper"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + self.plasma.pressure_fast_perpendicular.values = Pbper.values + def setup_opt_data(self, phantoms=False, **kwargs): if not hasattr(self, "plasma"): raise ValueError("Missing plasma object required for setup_opt_data") @@ -371,6 +396,9 @@ def _exp_data(self, **kwargs): opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( opt_data["cxff_pi"]["ti"] != 0, ) + # opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( + # opt_data["cxff_pi"]["ti"].channel == 0, + # ) if "ts" in self.diagnostics: # TODO: fix error, for now flat error opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel >19, drop=True) @@ -475,6 +503,7 @@ def __call__( priors=DEFAULT_PRIORS, profile_params=DEFAULT_PROFILE_PARAMS, phantoms=True, + fast_particles=True, tstart=0.02, tend=0.10, dt=0.005, From b9d09e2d4dcea1b4f75990b28b944175b91b58c6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 12 Sep 2023 13:07:28 +0100 Subject: [PATCH 184/288] ASTRA options added to BDA --- indica/workflows/bayes_workflow_example.py | 72 +++++++++++++++++++--- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 9b5651a6..aac9d201 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -23,6 +23,7 @@ from indica.writers.bda_tree import create_nodes from indica.writers.bda_tree import write_nodes from indica.readers.read_st40 import ReadST40 +from indica.equilibrium import Equilibrium # global configurations @@ -128,11 +129,18 @@ def __init__( priors: dict, profile_params: dict, pulse: int = None, - phantoms: bool = False, - fast_particles = False, tstart=0.02, tend=0.10, dt=0.005, + + phantoms: bool = False, + fast_particles = False, + astra_run=None, + astra_pulse_range=13000000, + astra_equilibrium=False, + efit_revision = 0, + set_ts_profiles = False, + astra_wp = False, ): self.pulse = pulse self.diagnostics = diagnostics @@ -140,11 +148,19 @@ def __init__( self.opt_quantity = opt_quantity self.priors = priors self.profile_params = profile_params - self.phantoms = phantoms - self.fast_particles = fast_particles self.tstart = tstart self.tend = tend self.dt = dt + + self.phantoms = phantoms + self.fast_particles = fast_particles + self.astra_run= astra_run + self.astra_pulse_range = astra_pulse_range + self.astra_equilibrium = astra_equilibrium + self.efit_revision = efit_revision + self.set_ts_profiles = set_ts_profiles + self.astra_wp = astra_wp + self.model_kwargs = {} for attribute in [ @@ -178,6 +194,10 @@ def __init__( self.read_data( self.diagnostics, tstart=self.tstart, tend=self.tend, dt=self.dt ) + if self.efit_revision != 0: + self.reader.get_equilibrium(revision=efit_revision, ) + self.equilibrium = self.reader.equilibrium + self.setup_models(self.diagnostics) def setup_plasma( @@ -215,7 +235,7 @@ def setup_plasma( self.plasma.set_equilibrium(self.equilibrium) self.plasma.update_profiles(self.profile_params) if self.fast_particles: - self._init_fast_particles() + self._init_fast_particles(run=self.astra_run) self.plasma.build_atomic_data(calc_power_loss=False) self.save_phantom_profiles() @@ -269,6 +289,12 @@ def setup_models(self, diagnostics: list): model = ChargeExchange(name=diag, element="ar") model.set_transect_transform(transform) + elif diag == "cxff_tws_c": + transform = self.transforms[diag] + transform.set_equilibrium(self.equilibrium) + model = ChargeExchange(name=diag, element="c") + model.set_transect_transform(transform) + elif diag == "ts": transform = self.transforms[diag] transform.set_equilibrium(self.equilibrium) @@ -279,9 +305,14 @@ def setup_models(self, diagnostics: list): raise ValueError(f"{diag} not found in setup_models") self.models[diag] = model - def _init_fast_particles(self): - st40_code = ReadST40(13000000 + self.pulse, self.tstart, self.tend, dt=self.dt, tree="astra") - st40_code.get_raw_data("", "astra", "RUN602") + def _init_fast_particles(self, run="RUN602", ): + st40_code = ReadST40(self.astra_pulse_range + self.pulse, self.tstart, self.tend, dt=self.dt, tree="astra") + astra_data = st40_code.get_raw_data("", "astra", run) + + if self.astra_equilibrium: + self.equilibrium = Equilibrium(astra_data) + self.plasma.equilibrium = self.equilibrium + st40_code.bin_data_in_time(["astra"], self.tstart, self.tend, self.dt) code_data = st40_code.binned_data["astra"] Nf = ( @@ -298,6 +329,16 @@ def _init_fast_particles(self): self.plasma.pressure_fast_parallel.values = Pblon.values Pbper = code_data["pbper"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) self.plasma.pressure_fast_perpendicular.values = Pbper.values + self.astra_data = code_data + + if self.set_ts_profiles: + overwritten_params = [param for param in self.param_names if any(xs in param for xs in ["Te", "Ne"])] + if any(overwritten_params): + raise ValueError(f"Te/Ne set by TS but then the following params overwritten: {overwritten_params}") + Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + self.plasma.electron_temperature.values = Te.values + Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + self.plasma.electron_density.values = Ne.values def setup_opt_data(self, phantoms=False, **kwargs): if not hasattr(self, "plasma"): @@ -350,6 +391,8 @@ def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) ) + # TODO: add chers + if noise: #TODO: add TS opt_data["smmh1.ne"] = opt_data["smmh1.ne"] + opt_data[ @@ -372,6 +415,7 @@ def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): opt_data["efit.wp"] = opt_data["efit.wp"] + opt_data[ "efit.wp" ].max().values * np.random.normal(0, noise_factor, None) + return opt_data def _exp_data(self, **kwargs): @@ -396,9 +440,18 @@ def _exp_data(self, **kwargs): opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( opt_data["cxff_pi"]["ti"] != 0, ) + # opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( # opt_data["cxff_pi"]["ti"].channel == 0, # ) + if "cxff_tws_c" in self.diagnostics: + opt_data["cxff_tws_c"]["ti"] = opt_data["cxff_tws_c"]["ti"].where( + opt_data["cxff_tws_c"]["ti"] != 0, + ) + # opt_data["cxff_tws_c"]["ti"] = opt_data["cxff_tws_c"]["ti"].where( + # opt_data["cxff_tws_c"]["ti"].channel == 1, + # ) + if "ts" in self.diagnostics: # TODO: fix error, for now flat error opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel >19, drop=True) @@ -407,6 +460,9 @@ def _exp_data(self, **kwargs): opt_data["ts.te"]["error"] = opt_data["ts.te"] * 0.10 + 10 opt_data["ts.ne"]["error"] = opt_data["ts.ne"] * 0.10 + 10 + if self.astra_wp: + opt_data["efit.wp"] = self.astra_data["wth"] + return opt_data def setup_optimiser( From 4089d41004b1e4a6142f85d937553d11545f36c7 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 12 Sep 2023 13:07:54 +0100 Subject: [PATCH 185/288] chers added --- indica/workflows/bayes_plots.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 127c0cdd..9d859960 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -92,6 +92,7 @@ def _plot_1d( xlabel="[]", xlim=None, figsize=(6.4, 4.8), + hide_legend=False, **kwargs, ): if blobkey not in blobs.keys(): @@ -140,11 +141,14 @@ def _plot_1d( label=f"{blobkey} data", zorder=4, ) + + plt.gca().set_ylim(bottom=0) set_axis_sci() plt.ylabel(ylabel) plt.xlabel(xlabel) plt.xlim(xlim) - plt.legend() + if not hide_legend: + plt.legend() plt.savefig(figheader + filename) plt.close() @@ -181,8 +185,8 @@ def violinplot( violin["bodies"][0].set_edgecolor("black") axs.set_xlabel(key) top = axs.get_ylim()[1] - bot = axs.get_ylim()[0] - axs.set_ylim(top=top * 1.1, bottom=bot * 0.9) + # bot = axs.get_ylim()[0] + axs.set_ylim(top=top * 1.1, bottom=0) axs.set_ylabel(f"{ylabel}") set_axis_sci() @@ -353,7 +357,22 @@ def plot_bayes_result( figheader=figheader, ylabel="Temperature [eV]", xlabel="Channel", + hide_legend=True, + ) + key = "CXFF_TWS_C.TI" + if key in model_data.keys(): + _plot_1d( + model_data, + key, + diag_data, + f"{key.replace('.', '_')}" + filetype, + figheader=figheader, + ylabel="Temperature [eV]", + xlabel="Channel", + hide_legend=True, + ) + key = "TS.TE" if key in model_data.keys(): _plot_1d( @@ -473,7 +492,7 @@ def plot_bayes_result( if __name__ == "__main__": - filehead = "./results/test/" + filehead = "./results/11312_ASTRA/" with open(filehead + "results.pkl", "rb") as handle: results = pickle.load(handle) plot_bayes_result(results, filehead, filetype=".png") From e74ed9a16c10e3ad2f127591258371367cb40844 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 12 Sep 2023 13:08:08 +0100 Subject: [PATCH 186/288] chers added --- indica/workflows/bayes_plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 9d859960..2b35598e 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -492,7 +492,7 @@ def plot_bayes_result( if __name__ == "__main__": - filehead = "./results/11312_ASTRA/" + filehead = "./results/test/" with open(filehead + "results.pkl", "rb") as handle: results = pickle.load(handle) plot_bayes_result(results, filehead, filetype=".png") From f4bc98012073475d8c8108d53112ecc9ec1c3fab Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 13 Sep 2023 15:10:32 +0100 Subject: [PATCH 187/288] Extra ASTRA options implemented --- indica/workflows/bayes_workflow_example.py | 48 ++++++++++++---------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index aac9d201..32ae1551 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -36,10 +36,10 @@ "Ne_prof.peaking": 1.2, "Nimp_prof.y0": 1e17, - "Nimp_prof.y1": 5e15, + "Nimp_prof.y1": 1e15, "Nimp_prof.yend": 1e15, "Nimp_prof.wcenter": 0.3, - "Nimp_prof.wped": 6, + "Nimp_prof.wped": 3, "Nimp_prof.peaking": 2, "Te_prof.y0": 3000, @@ -59,14 +59,14 @@ DEFAULT_PRIORS = { "Ne_prof.y0": get_uniform(2e19, 4e20), - "Ne_prof.y1": get_uniform(1e18, 1e19), + "Ne_prof.y1": get_uniform(1e18, 2e19), "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), - "Ne_prof.wped": get_uniform(2, 6), + "Ne_prof.wped": get_uniform(2, 20), "Ne_prof.wcenter": get_uniform(0.2, 0.4), "Ne_prof.peaking": get_uniform(1, 4), - "Nimp_prof.y0": loguniform(1e16, 1e18), - "Nimp_prof.y1": get_uniform(1e15, 1e16), + "Nimp_prof.y0": loguniform(1e15, 1e18), + "Nimp_prof.y1": loguniform(1e14, 1e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 ), @@ -195,8 +195,8 @@ def __init__( self.diagnostics, tstart=self.tstart, tend=self.tend, dt=self.dt ) if self.efit_revision != 0: - self.reader.get_equilibrium(revision=efit_revision, ) - self.equilibrium = self.reader.equilibrium + efit_equilibrium = self.reader.reader_equil.get("", "efit", self.efit_revision) + self.equilibrium = Equilibrium(efit_equilibrium) self.setup_models(self.diagnostics) @@ -261,13 +261,13 @@ def setup_models(self, diagnostics: list): # machine_dimensions=machine_dims, # passes=2, # ) - los_transform.set_equilibrium(self.equilibrium) + los_transform.set_equilibrium(self.equilibrium, force=True) model = Interferometry(name=diag) model.set_los_transform(los_transform) elif diag == "xrcs": los_transform = self.transforms[diag] - los_transform.set_equilibrium(self.equilibrium) + los_transform.set_equilibrium(self.equilibrium, force=True) window = None if hasattr(self, "data"): if diag in self.data.keys(): @@ -285,19 +285,19 @@ def setup_models(self, diagnostics: list): elif diag == "cxff_pi": transform = self.transforms[diag] - transform.set_equilibrium(self.equilibrium) + transform.set_equilibrium(self.equilibrium, force=True) model = ChargeExchange(name=diag, element="ar") model.set_transect_transform(transform) elif diag == "cxff_tws_c": transform = self.transforms[diag] - transform.set_equilibrium(self.equilibrium) + transform.set_equilibrium(self.equilibrium, force=True) model = ChargeExchange(name=diag, element="c") model.set_transect_transform(transform) elif diag == "ts": transform = self.transforms[diag] - transform.set_equilibrium(self.equilibrium) + transform.set_equilibrium(self.equilibrium, force=True) model = ThomsonScattering(name=diag, ) model.set_transect_transform(transform) @@ -306,7 +306,7 @@ def setup_models(self, diagnostics: list): self.models[diag] = model def _init_fast_particles(self, run="RUN602", ): - st40_code = ReadST40(self.astra_pulse_range + self.pulse, self.tstart, self.tend, dt=self.dt, tree="astra") + st40_code = ReadST40(self.astra_pulse_range + self.pulse, self.tstart-self.dt, self.tend+self.dt, dt=self.dt, tree="astra") astra_data = st40_code.get_raw_data("", "astra", run) if self.astra_equilibrium: @@ -441,16 +441,20 @@ def _exp_data(self, **kwargs): opt_data["cxff_pi"]["ti"] != 0, ) - # opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( - # opt_data["cxff_pi"]["ti"].channel == 0, - # ) + opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( + opt_data["cxff_pi"]["ti"].channel > 3, + ) + opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( + opt_data["cxff_pi"]["ti"].channel < 6, + ) + if "cxff_tws_c" in self.diagnostics: opt_data["cxff_tws_c"]["ti"] = opt_data["cxff_tws_c"]["ti"].where( opt_data["cxff_tws_c"]["ti"] != 0, ) - # opt_data["cxff_tws_c"]["ti"] = opt_data["cxff_tws_c"]["ti"].where( - # opt_data["cxff_tws_c"]["ti"].channel == 1, - # ) + opt_data["cxff_tws_c"]["ti"] = opt_data["cxff_tws_c"]["ti"].where( + opt_data["cxff_tws_c"]["ti"].channel < 2, + ) if "ts" in self.diagnostics: # TODO: fix error, for now flat error @@ -559,7 +563,7 @@ def __call__( priors=DEFAULT_PRIORS, profile_params=DEFAULT_PROFILE_PARAMS, phantoms=True, - fast_particles=True, + fast_particles=False, tstart=0.02, tend=0.10, dt=0.005, @@ -577,6 +581,6 @@ def __call__( mds_write=True, plot=True, burn_frac=0.10, - iterations=100, + iterations=2000, ) From 875293cba157efc259f1ea0158ca7d91310a4299 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 13 Sep 2023 15:36:14 +0100 Subject: [PATCH 188/288] prior for ne.wped increased to 30 --- indica/workflows/bayes_workflow_example.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 32ae1551..b9354b05 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -61,7 +61,7 @@ "Ne_prof.y0": get_uniform(2e19, 4e20), "Ne_prof.y1": get_uniform(1e18, 2e19), "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), - "Ne_prof.wped": get_uniform(2, 20), + "Ne_prof.wped": get_uniform(2, 30), "Ne_prof.wcenter": get_uniform(0.2, 0.4), "Ne_prof.peaking": get_uniform(1, 4), @@ -459,10 +459,10 @@ def _exp_data(self, **kwargs): if "ts" in self.diagnostics: # TODO: fix error, for now flat error opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel >19, drop=True) - opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel >19, drop=True) + opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel > 19, drop=True) - opt_data["ts.te"]["error"] = opt_data["ts.te"] * 0.10 + 10 - opt_data["ts.ne"]["error"] = opt_data["ts.ne"] * 0.10 + 10 + opt_data["ts.te"]["error"] = opt_data["ts.te"] * 0.10 + 50 + opt_data["ts.ne"]["error"] = opt_data["ts.ne"] * 0.10 if self.astra_wp: opt_data["efit.wp"] = self.astra_data["wth"] From 6545540fdd69f1fb410a62b34a5815d7975b5504 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 13 Sep 2023 15:36:44 +0100 Subject: [PATCH 189/288] not automatically adding equil to transforms now --- indica/readers/read_st40.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indica/readers/read_st40.py b/indica/readers/read_st40.py index 62cdaf62..f14f999e 100644 --- a/indica/readers/read_st40.py +++ b/indica/readers/read_st40.py @@ -120,8 +120,8 @@ def get_raw_data(self, uid: str, instrument: str, revision: RevisionLike = 0): continue transform = data[quant].transform - if hasattr(transform, "set_equilibrium"): - transform.set_equilibrium(self.equilibrium) + # if hasattr(transform, "set_equilibrium"): + # transform.set_equilibrium(self.equilibrium) self.transforms[instrument] = transform self.raw_data[instrument] = data From fd859b54cccc0e65750380c640c594b439ff7baf Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 21 Sep 2023 13:44:13 +0100 Subject: [PATCH 190/288] Fixed plotting for multiple time points --- indica/workflows/bayes_workflow_example.py | 220 ++++++++++++++++++--- 1 file changed, 188 insertions(+), 32 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index b9354b05..a4307e33 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -1,10 +1,11 @@ -from typing import Any +from typing import Any, List, Tuple from typing import Dict import emcee import flatdict import numpy as np from scipy.stats import loguniform +import xarray as xr from indica.bayesmodels import BayesModels from indica.bayesmodels import get_uniform @@ -61,7 +62,7 @@ "Ne_prof.y0": get_uniform(2e19, 4e20), "Ne_prof.y1": get_uniform(1e18, 2e19), "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), - "Ne_prof.wped": get_uniform(2, 30), + "Ne_prof.wped": get_uniform(2, 6), "Ne_prof.wcenter": get_uniform(0.2, 0.4), "Ne_prof.peaking": get_uniform(1, 4), @@ -87,14 +88,14 @@ "Ti_prof.wped": get_uniform(2, 6), "Ti_prof.wcenter": get_uniform(0.2, 0.4), "Ti_prof.peaking": get_uniform(1, 6), - "xrcs.pixel_offset": get_uniform(-4.01, -3.99), + "xrcs.pixel_offset": get_uniform(-4.01, -4.0), } OPTIMISED_PARAMS = [ "Ne_prof.y1", "Ne_prof.y0", "Ne_prof.peaking", - "Ne_prof.wcenter", + # "Ne_prof.wcenter", "Ne_prof.wped", # "Nimp_prof.y1", "Nimp_prof.y0", @@ -102,17 +103,17 @@ # "Nimp_prof.wped", "Nimp_prof.peaking", "Te_prof.y0", - "Te_prof.wped", + # "Te_prof.wped", "Te_prof.wcenter", "Te_prof.peaking", "Ti_prof.y0", - "Ti_prof.wped", + # "Ti_prof.wped", "Ti_prof.wcenter", "Ti_prof.peaking", ] OPTIMISED_QUANTITY = [ "xrcs.spectra", - # "cxff_pi.ti", + "cxff_pi.ti", "efit.wp", # "smmh1.ne", "ts.te", @@ -140,6 +141,7 @@ def __init__( astra_equilibrium=False, efit_revision = 0, set_ts_profiles = False, + set_all_profiles=False, astra_wp = False, ): self.pulse = pulse @@ -159,6 +161,7 @@ def __init__( self.astra_equilibrium = astra_equilibrium self.efit_revision = efit_revision self.set_ts_profiles = set_ts_profiles + self.set_all_profiles = set_all_profiles self.astra_wp = astra_wp self.model_kwargs = {} @@ -205,7 +208,7 @@ def setup_plasma( tstart=None, tend=None, dt=None, - tsample=0.050, + tsample=None, main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), @@ -228,14 +231,20 @@ def setup_plasma( full_run=False, n_rad=n_rad, ) - self.tsample = tsample - self.plasma.time_to_calculate = self.plasma.t[ - np.abs(tsample - self.plasma.t).argmin() - ] + + if tsample == None: + self.tsample = self.plasma.t + else: + self.tsample = self.plasma.t[ + np.abs(tsample - self.plasma.t).argmin() + ] + + self.plasma.time_to_calculate = self.tsample self.plasma.set_equilibrium(self.equilibrium) self.plasma.update_profiles(self.profile_params) if self.fast_particles: self._init_fast_particles(run=self.astra_run) + self.plasma.update_profiles({}) self.plasma.build_atomic_data(calc_power_loss=False) self.save_phantom_profiles() @@ -306,6 +315,7 @@ def setup_models(self, diagnostics: list): self.models[diag] = model def _init_fast_particles(self, run="RUN602", ): + st40_code = ReadST40(self.astra_pulse_range + self.pulse, self.tstart-self.dt, self.tend+self.dt, dt=self.dt, tree="astra") astra_data = st40_code.get_raw_data("", "astra", run) @@ -335,10 +345,23 @@ def _init_fast_particles(self, run="RUN602", ): overwritten_params = [param for param in self.param_names if any(xs in param for xs in ["Te", "Ne"])] if any(overwritten_params): raise ValueError(f"Te/Ne set by TS but then the following params overwritten: {overwritten_params}") - Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - self.plasma.electron_temperature.values = Te.values - Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - self.plasma.electron_density.values = Ne.values + Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 + self.plasma.Te_prof = lambda: Te.values + Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 + self.plasma.Ne_prof = lambda: Ne.values + + if self.set_all_profiles: + overwritten_params = [param for param in self.param_names if any(xs in param for xs in ["Te", "Ti", "Ne", "Nimp"])] + if any(overwritten_params): + raise ValueError(f"Te/Ne set by TS but then the following params overwritten: {overwritten_params}") + Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 + self.plasma.Te_prof = lambda: Te.values + Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 + self.plasma.Ne_prof = lambda: Ne.values + Ti = code_data["ti"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 + self.plasma.Ti_prof = lambda: Ti.values + Nimp = code_data["niz1"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 + self.plasma.Nimp_prof = lambda: Nimp.values def setup_opt_data(self, phantoms=False, **kwargs): if not hasattr(self, "plasma"): @@ -461,8 +484,8 @@ def _exp_data(self, **kwargs): opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel >19, drop=True) opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel > 19, drop=True) - opt_data["ts.te"]["error"] = opt_data["ts.te"] * 0.10 + 50 - opt_data["ts.ne"]["error"] = opt_data["ts.ne"] * 0.10 + opt_data["ts.te"]["error"] = opt_data["ts.te"].max(dim="channel") * 0.05 + opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 if self.astra_wp: opt_data["efit.wp"] = self.astra_data["wth"] @@ -474,13 +497,13 @@ def setup_optimiser( model_kwargs, nwalkers=50, burn_frac=0.10, - sample_high_density=False, + sample_method="random", **kwargs, ): self.model_kwargs = model_kwargs self.nwalkers = nwalkers self.burn_frac = burn_frac - self.sample_high_density = sample_high_density + self.sample_method = sample_method self.bayesmodel = BayesModels( plasma=self.plasma, @@ -500,21 +523,106 @@ def setup_optimiser( moves=self.move, kwargs=model_kwargs, ) - self.start_points = self._sample_start_points( - sample_high_density=self.sample_high_density, **kwargs - ) - def _sample_start_points(self, sample_high_density: bool = True, nsamples=100, **kwargs): - if sample_high_density: + + + def _sample_start_points(self, sample_method: str = "random", nsamples=100, **kwargs): + if sample_method == "high_density": start_points = self.bayesmodel.sample_from_high_density_region( self.param_names, self.sampler, self.nwalkers, nsamples=nsamples ) + + elif sample_method == "ga": + self.start_points = self._sample_start_points(sample_method="random", **kwargs) + samples_in_weird_format = self.ga_opt(**kwargs) + sample_start_points = np.array([idx[1] for idx in samples_in_weird_format]) + + start_points = np.random.normal( + np.mean(sample_start_points, axis=0), + np.std(sample_start_points, axis=0), + size=(self.nwalkers, sample_start_points.shape[1]), + ) + + + elif sample_method == "random": + start_points = self.bayesmodel.sample_from_priors( + self.param_names, size=self.nwalkers + ) + else: + print(f"Sample method: {sample_method} not recognised, Defaulting to random sampling") start_points = self.bayesmodel.sample_from_priors( self.param_names, size=self.nwalkers ) return start_points + + def ga_opt(self, num_gens=30, popsize=50, sols_to_return=5, mutation_probability=None, **kwargs) -> list(tuple((float, []))): + """Runs the GA optimization, and returns a number of the best solutions. Uses + a population convergence stopping criteria: fitness does not improve in 3 successive generations, we stop. + + Args: + num_gens (int, optional): Maximum number of generations to run. Defaults to 30. + popsize (int, optional): Population size. Defaults to 50. + sols_to_return (int, optional): How many of the best solutions the function shall return. Defaults to 5. + + Returns: + list(tuple(float, np.Array(float))): list of tuples, where first element is fitness, second np.array of the parameters. + """ + + import pygad + import time + + # Packaged evaluation function + def idiot_proof(ga_instance, x, sol_idx): + res, _ = self.bayesmodel.ln_posterior(dict(zip(self.param_names, x))) + # print(-res) + return float(res) + + print(f"Running GA for a maximum of {num_gens} generations of {popsize} individuals each.") + + # Initialize the GA instance + ga_instance = pygad.GA(num_generations=num_gens, + num_parents_mating=20, + sol_per_pop=popsize, + num_genes=len(self.start_points[0]), + fitness_func=idiot_proof, + initial_population=self.start_points, + save_best_solutions=True, + stop_criteria="saturate_5", + mutation_probability=mutation_probability) + + st = time.time() + # Execute + ga_instance.run() + + print( + f"Time ran: {time.time() - st:.2f} seconds. Ran total of {ga_instance.generations_completed} generations.") + + # Saves the fitness evolution plot + # figure = ga_instance.plot_fitness() + # figure.savefig(f'GA_plot.png', dpi=300) + + # Organizing all the non-inf individuals from the last generation + feasible_indices = [i for i in range(len(ga_instance.last_generation_fitness)) if + ga_instance.last_generation_fitness[i] != -np.inf] + feasible_individuals = [ga_instance.population[i] for i in feasible_indices] + feasible_fitnesses = [ga_instance.last_generation_fitness[i] for i in feasible_indices] + + # for i, item in enumerate(feasible_individuals_with_keywords): + # item["fitness"]=feasible_fitnesses[i + # feasible_individuals_with_keywords=sorted(feasible_individuals_with_keywords,key= lambda d:d['fitness'],reverse=True) + # feasible_individuals_and_fitnesses=[tuple(feasible_fitnesses[i],feasible_individuals[i]) for i in len(feasible_individuals)] + + # Combining the last individuals to a collection and sorting + feasible_individuals_and_fitnesses = [] + for i in range(len(feasible_fitnesses)): + feasible_individuals_and_fitnesses.append(tuple((feasible_fitnesses[i], feasible_individuals[i]))) + feasible_individuals_and_fitnesses = sorted(feasible_individuals_and_fitnesses, key=lambda x: x[0], + reverse=True) + + return feasible_individuals_and_fitnesses[:sols_to_return] + def __call__( self, filepath="./results/test/", @@ -526,6 +634,10 @@ def __call__( burn_frac=0.10, **kwargs, ): + + self.iterations = iterations + self.burn_frac = burn_frac + if mds_write: # check_analysis_run(self.pulse, self.run) self.node_structure = create_nodes( @@ -535,11 +647,54 @@ def __call__( mode="EDIT", ) - self.run_sampler(iterations=iterations, burn_frac=burn_frac) - self.save_pickle(filepath=filepath) + self.result = self._build_inputs_dict() + results = [] + + if not self.tsample.shape: + self.tsample = np.array([self.tsample]) + + self.plasma.time_to_calculate = self.tsample[0] + self.start_points = self._sample_start_points( + sample_method=self.sample_method, **kwargs + ) + + for t in self.tsample: + self.plasma.time_to_calculate = t + print(f"Time: {t}") + self.run_sampler(iterations=iterations, burn_frac=burn_frac) + _result = self._build_result_dict() + results.append(_result) - if plot: # currently requires result with DataArrays - plot_bayes_result(self.result, filepath) + self.start_points = self.sampler.get_chain()[-1,:,:] + self.sampler.reset() + + _result = dict(_result, ** self.result) + + self.save_pickle(_result, filepath=filepath, ) + + if plot: # currently requires result with DataArrays + plot_bayes_result(_result, filepath) + + self.result = dict(self.result, ** results[-1]) + profiles = {} + globals = {} + for key, prof in results[0]["PROFILES"].items(): + if key == "RHO_POLOIDAL": + profiles[key] = results[0]["PROFILES"]["RHO_POLOIDAL"] + elif key == "RHO_TOR": + profiles[key] = results[0]["PROFILES"]["RHO_TOR"] + else: + _profs = [result["PROFILES"][key] for result in results] + profiles[key] = xr.concat(_profs, self.tsample) + + + for key, prof in results[0]["GLOBAL"].items(): + _glob = [result["GLOBAL"][key] for result in results] + globals[key] = xr.concat(_glob, self.tsample) + + result = {"PROFILES":profiles, "GLOBAL":globals} + + self.result = dict(self.result, **result,) self.result = self.dict_of_dataarray_to_numpy(self.result) if mds_write: @@ -573,14 +728,15 @@ def __call__( tsample=0.05, ) run.setup_opt_data(phantoms=run.phantoms) - run.setup_optimiser(nwalkers=50, sample_high_density=True, model_kwargs=run.model_kwargs) + run.setup_optimiser(nwalkers=50, sample_method="high_density", model_kwargs=run.model_kwargs, nsamples=100) + # run.setup_optimiser(nwalkers=50, sample_method="ga", model_kwargs=run.model_kwargs, num_gens=50, popsize=100, sols_to_return=3, mutation_probability=None) results = run( filepath=f"./results/test/", pulse_to_write=25000000, run="RUN01", mds_write=True, plot=True, - burn_frac=0.10, - iterations=2000, + burn_frac=0.0, + iterations=500, ) From 01ea6b4c08a18dfe9faf27bdf388295dbfd7df5f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 21 Sep 2023 13:45:11 +0100 Subject: [PATCH 191/288] save_pickle now takes dictionary as argument --- indica/workflows/abstract_bayes_workflow.py | 126 +++++++++++--------- 1 file changed, 68 insertions(+), 58 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 133d8b8c..7f5c75e5 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -132,13 +132,13 @@ def save_phantom_profiles(self, kinetic_profiles=None): self.phantom_profiles = phantom_profiles - def _build_result_dict(self): + def _build_inputs_dict(self): """ Returns ------- - dictionary of results in MDS+ structure + dictionary of inputs in MDS+ structure """ @@ -181,24 +181,33 @@ def _build_result_dict(self): for diag_name in self.diagnostics } - result["MODEL_DATA"] = { + result["DIAG_DATA"] = { diag_name.upper(): { - quantity[1].upper(): self.blobs[f"{quantity[0]}.{quantity[1]}"] + quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] for quantity in quant_list if quantity[0] == diag_name } for diag_name in self.diagnostics } - result["MODEL_DATA"]["SAMPLES"] = self.samples - result["DIAG_DATA"] = { + + return result + + + def _build_result_dict(self, ): + result = {} + quant_list = [item.split(".") for item in self.opt_quantity] + + result["MODEL_DATA"] = { diag_name.upper(): { - quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] + quantity[1].upper(): self.blobs[f"{quantity[0]}.{quantity[1]}"] for quantity in quant_list if quantity[0] == diag_name } for diag_name in self.diagnostics } + result["MODEL_DATA"]["SAMPLES"] = self.samples + result["PHANTOMS"] = { "FLAG": self.phantoms, @@ -235,22 +244,22 @@ def _build_result_dict(self): "RHO_TOR": self.plasma.equilibrium.rhotor.interp(t=self.plasma.t), "NE": self.blobs["electron_density"].median(dim="index"), "NI": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .median(dim="index"), "TE": self.blobs["electron_temperature"].median(dim="index"), "TI": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .median(dim="index"), "NFAST": self.blobs["fast_density"].median(dim="index"), "NNEUTR": self.blobs["neutral_density"].median(dim="index"), "NE_ERR": self.blobs["electron_density"].std(dim="index"), "NI_ERR": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .std(dim="index"), "TE_ERR": self.blobs["electron_temperature"].std(dim="index"), "TI_ERR": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .std(dim="index"), + .sel(element=self.plasma.main_ion) + .std(dim="index"), "NFAST_ERR": self.blobs["fast_density"].std(dim="index"), "NNEUTR_ERR": self.blobs["neutral_density"].std(dim="index"), "ZEFF": self.blobs["zeff"].sum("element").median(dim="index"), @@ -260,8 +269,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"NIZ{num_imp + 1}": self.blobs["impurity_density"] - .sel(element=imp) - .median(dim="index") + .sel(element=imp) + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -269,8 +278,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"NIZ{num_imp + 1}_ERR": self.blobs["impurity_density"] - .sel(element=imp) - .std(dim="index") + .sel(element=imp) + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -278,8 +287,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"TIZ{num_imp + 1}": self.blobs["ion_temperature"] - .sel(element=imp) - .median(dim="index") + .sel(element=imp) + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -287,8 +296,8 @@ def _build_result_dict(self): **result["PROFILES"], **{ f"TIZ{num_imp + 1}_ERR": self.blobs["ion_temperature"] - .sel(element=imp) - .std(dim="index") + .sel(element=imp) + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -328,27 +337,27 @@ def _build_result_dict(self): result["GLOBAL"] = { "TI0": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "TE0": self.blobs["electron_temperature"] - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "NE0": self.blobs["electron_density"] - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "NI0": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index"), + .sel(element=self.plasma.main_ion) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index"), "WP": self.blobs["wp"] - .median(dim="index"), + .median(dim="index"), "WP_ERR": self.blobs["wp"] - .std(dim="index"), + .std(dim="index"), "WTH": self.blobs["wth"] - .median(dim="index"), + .median(dim="index"), "WTH_ERR": self.blobs["wth"] - .std(dim="index"), + .std(dim="index"), "PTOT": self.blobs["ptot"] .median(dim="index"), "PTOT_ERR": self.blobs["ptot"] @@ -363,9 +372,9 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"TI0Z{num_imp + 1}": self.blobs["ion_temperature"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -373,9 +382,9 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"TI0Z{num_imp + 1}_ERR": self.blobs["ion_temperature"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -383,9 +392,9 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"NI0Z{num_imp + 1}": self.blobs["impurity_density"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .median(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .median(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } @@ -393,15 +402,13 @@ def _build_result_dict(self): **result["GLOBAL"], **{ f"NI0Z{num_imp + 1}_ERR": self.blobs["impurity_density"] - .sel(element=imp) - .sel(rho_poloidal=0, method="nearest") - .std(dim="index") + .sel(element=imp) + .sel(rho_poloidal=0, method="nearest") + .std(dim="index") for num_imp, imp in enumerate(self.plasma.impurities) }, } - - self.result = result - return self.result + return result def run_sampler(self, iterations, burn_frac): """ @@ -424,10 +431,13 @@ def run_sampler(self, iterations, burn_frac): self.param_names.__len__(), auto_sample=10, ) - blobs = self.sampler.get_blobs(discard=int(iterations * burn_frac), flat=True) - blob_names = self.sampler.get_blobs().flatten()[0].keys() + _blobs = self.sampler.get_blobs(discard=int(iterations * burn_frac), flat=True) + blobs = [blob for blob in _blobs if blob] # remove empty blobs + + blob_names = blobs[0].keys() self.samples = np.arange(0, blobs.__len__()) + self.blobs = { blob_name: xr.concat( [data[blob_name] for data in blobs], @@ -440,13 +450,12 @@ def run_sampler(self, iterations, burn_frac): self.param_names, size=int(1e4) ) self.post_sample = self.sampler.get_chain(discard=int(iterations * burn_frac), flat=True) - self.result = self._build_result_dict() - def save_pickle(self, filepath): + def save_pickle(self, result, filepath): if filepath: Path(filepath).mkdir(parents=True, exist_ok=True) with open(filepath + "results.pkl", "wb") as handle: - pickle.dump(self.result, handle) + pickle.dump(result, handle) def dict_of_dataarray_to_numpy(self, dict_of_dataarray): """ @@ -469,13 +478,14 @@ def __call__(self, filepath="./results/test/", **kwargs): return self.result -def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sample=5): +def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sample=5,): autocorr = np.ones(shape=(iterations, n_params)) * np.nan old_tau = np.inf for sample in sampler.sample( start_points, iterations=iterations, progress=True, + skip_initial_state_check=True, ): if sampler.iteration % auto_sample: continue From ada6ce03a75bc6d90581c7e3b3b8e53668c092b1 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 21 Sep 2023 13:45:46 +0100 Subject: [PATCH 192/288] stashing --- indica/bayesmodels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py index 102cdd65..b6b5f74b 100644 --- a/indica/bayesmodels.py +++ b/indica/bayesmodels.py @@ -214,7 +214,7 @@ def sample_from_high_density_region( while samples.size < param_names.__len__() * nwalkers: sample = np.random.normal( np.mean(best_start_points, axis=0), - best_points_std * 2, + best_points_std, size=(nwalkers * 5, len(param_names)), ) start = {name: sample[:, idx] for idx, name in enumerate(param_names)} From 5609b93b36a1d5fb4231d69e8aa656f63876d395 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 21 Sep 2023 13:49:48 +0100 Subject: [PATCH 193/288] stashing --- indica/workflows/bayes_plots.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 2b35598e..15d61da1 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -274,12 +274,9 @@ def plot_bayes_result( diag_data["CXFF_PI.TI0"] = diag_data["CXFF_PI.TI"].sel( channel=max_channel ) - # model_data["CXFF_PI.TI0"] = model_data["CXFF_PI.TI"].sel( - # channel=diag_data["CXFF_PI.TI"].channel - # ) - # diag_data["CXFF_PI.TI0"] = diag_data["CXFF_PI.TI"].sel( - # channel=diag_data["CXFF_PI.TI"].channel - # ) + model_data["CXFF_PI.TI0"] = model_data["CXFF_PI.TI"].sel( + channel=diag_data["CXFF_PI.TI"].channel + ) key = "CXFF_PI.TI0" if key in model_data.keys(): From ecdbb65ccce913ae72b16ab20cffbead3db5bfc7 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 21 Sep 2023 16:52:20 +0100 Subject: [PATCH 194/288] Rough batch script for running BDA --- indica/workflows/batch_bda.py | 287 ++++++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 indica/workflows/batch_bda.py diff --git a/indica/workflows/batch_bda.py b/indica/workflows/batch_bda.py new file mode 100644 index 00000000..ef698ec1 --- /dev/null +++ b/indica/workflows/batch_bda.py @@ -0,0 +1,287 @@ +from indica.workflows.bayes_workflow_example import BayesWorkflowExample, DEFAULT_PRIORS, DEFAULT_PROFILE_PARAMS + +PARAMS_SET_TS =[ + # "Ne_prof.y1", + # "Ne_prof.y0", + # "Ne_prof.peaking", + # "Ne_prof.wcenter", + # "Ne_prof.wped", + # "Nimp_prof.y1", + "Nimp_prof.y0", + "Nimp_prof.wcenter", + # "Nimp_prof.wped", + "Nimp_prof.peaking", + # "Te_prof.y0", + # "Te_prof.wped", + # "Te_prof.wcenter", + # "Te_prof.peaking", + "Ti_prof.y0", + # # "Ti_prof.wped", + "Ti_prof.wcenter", + "Ti_prof.peaking", + "xrcs.pixel_offset", + ] + +PARAMS_DEFAULT =[ + "Ne_prof.y1", + "Ne_prof.y0", + "Ne_prof.peaking", + # "Ne_prof.wcenter", + "Ne_prof.wped", + # "Nimp_prof.y1", + "Nimp_prof.y0", + # "Nimp_prof.wcenter", + # "Nimp_prof.wped", + "Nimp_prof.peaking", + "Te_prof.y0", + # "Te_prof.wped", + # "Te_prof.wcenter", + "Te_prof.peaking", + "Ti_prof.y0", + # "Ti_prof.wped", + "Ti_prof.wcenter", + "Ti_prof.peaking", + "xrcs.pixel_offset", + ] + +PARAMS_SET_ALL =[ + # "Ne_prof.y1", + # "Ne_prof.y0", + # "Ne_prof.peaking", + # "Ne_prof.wcenter", + # "Ne_prof.wped", + # "Nimp_prof.y1", + # "Nimp_prof.y0", + # "Nimp_prof.wcenter", + # "Nimp_prof.wped", + # "Nimp_prof.peaking", + # "Te_prof.y0", + # "Te_prof.wped", + # "Te_prof.wcenter", + # "Te_prof.peaking", + # "Ti_prof.y0", + # "Ti_prof.wped", + # "Ti_prof.wcenter", + # "Ti_prof.peaking", + "xrcs.pixel_offset", + ] + + +DIAG_DEFAULT = [ + "xrcs", + "ts", + "efit", + "cxff_pi", + "cxff_tws_c" + # "smmh1", + ] + +DIAG_DEFAULT_CHERS = [ + "xrcs", + "ts", + "efit", + # "cxff_pi", + "cxff_tws_c" + # "smmh1", + ] + +DIAG_DEFAULT_PI = [ + "xrcs", + "ts", + "efit", + "cxff_pi", + # "cxff_tws_c" + # "smmh1", + ] + + +DIAG_SET_TS = [ + "xrcs", + # "ts", + "efit", + "cxff_pi", + "cxff_tws_c" + # "smmh1", + ] + +OPT_DEFAULT = [ + "xrcs.spectra", + "ts.ne", + "ts.te", + "efit.wp", + "cxff_pi.ti", + "cxff_tws_c.ti", + # "smmh1.ne" + ] + +OPT_SET_TS = [ + "xrcs.spectra", + # "ts.ne", + # "ts.te", + "efit.wp", + "cxff_pi.ti", + "cxff_tws_c.ti", + # "smmh1.ne" + ] + +def pulse_example(pulse, diagnostics, param_names, opt_quantity, t_opt, iterations=100, run="RUN01", + fast_particles=False, astra_run="RUN602", astra_pulse_range=13000000, + astra_equilibrium=False, efit_revision=0, set_ts_profiles=False, set_all_profiles=True, + astra_wp=False, **kwargs): + + workflow = BayesWorkflowExample( + pulse=pulse, + diagnostics=diagnostics, + param_names=param_names, + opt_quantity=opt_quantity, + priors=DEFAULT_PRIORS, + profile_params=DEFAULT_PROFILE_PARAMS, + phantoms=False, + fast_particles=fast_particles, + tstart=0.06, + tend=0.10, + dt=0.005, + astra_run=astra_run, + astra_pulse_range=astra_pulse_range, + astra_equilibrium=astra_equilibrium, + efit_revision=efit_revision, + set_ts_profiles = set_ts_profiles, + set_all_profiles=set_all_profiles, + astra_wp = astra_wp, + ) + + workflow.setup_plasma( + tsample=t_opt, + # n_rad=50 + ) + workflow.setup_opt_data(phantoms=workflow.phantoms) + workflow.setup_optimiser(nwalkers=40, sample_method="high_density", model_kwargs=workflow.model_kwargs, nsamples=100) + results = workflow( + filepath=f"./results/{workflow.pulse}.{run}/", + pulse_to_write=25000000 + workflow.pulse, + run=run, + mds_write=False, + plot=True, + burn_frac=0.20, + iterations=iterations, + ) + +# (11089, DIAG_DEFAULT_CHERS, PARAMS_DEFAULT, [ + # "xrcs.spectra", + # "ts.ne", + # "ts.te", + # "efit.wp", + # # "cxff_pi.ti", + # "cxff_tws_c.ti", + # # "smmh1.ne" + # ], 0.100, + # dict(run="RUN01", fast_particles=True, astra_run="RUN601", astra_pulse_range=13000000, astra_equilibrium=False, + # efit_revision=2)), + + + +if __name__ == "__main__": + + pulse_info = [ + + # (11211, DIAG_DEFAULT, PARAMS_DEFAULT, OPT_DEFAULT, 0.084), + # (11215, DIAG_DEFAULT_CHERS, PARAMS_DEFAULT, [ + # "xrcs.spectra", + # "ts.ne", + # "ts.te", + # "efit.wp", + # # "cxff_pi.ti", + # "cxff_tws_c.ti", + # # "smmh1.ne" + # ], 0.070, dict(fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), + # + # # (11224, DIAG_DEFAULT, PARAMS_DEFAULT, OPT_DEFAULT, 0.090), + # (11225, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ + # "xrcs.spectra", + # "ts.ne", + # "ts.te", + # "efit.wp", + # "cxff_pi.ti", + # # "cxff_tws_c.ti", + # # "smmh1.ne" + # ], 0.090, dict(efit_revision=2, fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), + # + # # (11226, DIAG_DEFAULT, PARAMS_DEFAULT, OPT_DEFAULT, 0.070), + # (11227, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ + # "xrcs.spectra", + # "ts.ne", + # "ts.te", + # "efit.wp", + # "cxff_pi.ti", + # # "cxff_tws_c.ti", + # # "smmh1.ne" + # ], 0.070, dict(fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), + # + # (11228, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ + # "xrcs.spectra", + # "ts.ne", + # "ts.te", + # "efit.wp", + # "cxff_pi.ti", + # # "cxff_tws_c.ti", + # # "smmh1.ne" + # ], 0.080, dict(run="RUN02", fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), + # (11238, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ + # "xrcs.spectra", + # "ts.ne", + # "ts.te", + # "efit.wp", + # "cxff_pi.ti", + # # "cxff_tws_c.ti", + # # "smmh1.ne" + # ], 0.075, dict(efit_revision=2, fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), + # + # (11312, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ + # "xrcs.spectra", + # "ts.ne", + # "ts.te", + # "efit.wp", + # "cxff_pi.ti", + # # "cxff_tws_c.ti", + # # "smmh1.ne" + # ], 0.080, dict(run="RUN01", fast_particles=True, astra_run="RUN14", astra_pulse_range=33000000, astra_equilibrium=True, set_ts_profiles=False, set_all_profiles=False, astra_wp=True)), + # + # (11314, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ + # "xrcs.spectra", + # "ts.ne", + # "ts.te", + # "efit.wp", + # "cxff_pi.ti", + # # "cxff_tws_c.ti", + # # "smmh1.ne" + # ], 0.080, dict(run="RUN01", fast_particles=True, astra_run="RUN3", astra_pulse_range=33000000, astra_equilibrium=True, set_ts_profiles=False, astra_wp=True)), + (11312, DIAG_SET_TS, PARAMS_SET_TS, [ + "xrcs.spectra", + # "ts.ne", + # "ts.te", + # "efit.wp", + "cxff_pi.ti", + # "cxff_tws_c.ti", + # "smmh1.ne" + ], 0.080, + dict(run="TEST_TI", fast_particles=True, astra_run="RUN14", astra_pulse_range=33000000, astra_equilibrium=True, + set_ts_profiles=True, set_all_profiles=False, astra_wp=True)), + + # Tree doesnt exist + ## (11317, 0.080, dict(run="RUN01", fast_particles=True, astra_run="RUN11", astra_pulse_range=33000000, astra_equilibrium=True, set_ts_profiles=True, astra_wp=True)), + ] + + # for info in pulse_info: + # if len(info) < 6: + # info = list(info) + # info.append(dict()) + # info = tuple(info) + # try: + # pulse_example(*info[0:5], iterations=2, **info[5]) + # except Exception as e: + # print(f"pulse: {info[0]}") + # print(repr(e)) + # continue + + for info in pulse_info: + pulse_example(*info[0:5], iterations=2000, **info[5]) \ No newline at end of file From cf736c7a557347b4f8566881661d92339bbe2c21 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 2 Oct 2023 14:41:50 +0100 Subject: [PATCH 195/288] sampler stopping condition based on moments added --- indica/workflows/abstract_bayes_workflow.py | 42 ++++++++++++++++++--- indica/workflows/bayes_workflow_example.py | 42 ++++++++++++++++++--- 2 files changed, 74 insertions(+), 10 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 24882966..20faa75c 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -9,7 +9,7 @@ from indica.equilibrium import fake_equilibrium from indica.readers.read_st40 import ReadST40 - +import scipy.stats as stats class AbstractBayesWorkflow(ABC): @abstractmethod @@ -423,14 +423,15 @@ def run_sampler(self, iterations, burn_frac): self.burn_frac = burn_frac self.iterations = iterations - self.autocorr = sample_with_autocorr( + self.autocorr = sample_with_moments( self.sampler, self.start_points, iterations, self.param_names.__len__(), auto_sample=10, + fractional_difference=0.05 ) - _blobs = self.sampler.get_blobs(discard=int(iterations * burn_frac), flat=True) + _blobs = self.sampler.get_blobs(discard=int(self.sampler.iteration * burn_frac), flat=True) blobs = [blob for blob in _blobs if blob] # remove empty blobs blob_names = blobs[0].keys() @@ -448,7 +449,7 @@ def run_sampler(self, iterations, burn_frac): self.prior_sample = self.bayesmodel.sample_from_priors( self.param_names, size=int(1e4) ) - self.post_sample = self.sampler.get_chain(discard=int(iterations * burn_frac), flat=True) + self.post_sample = self.sampler.get_chain(discard=int(self.sampler.iteration * burn_frac), flat=True) def save_pickle(self, result, filepath): if filepath: @@ -484,7 +485,7 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl start_points, iterations=iterations, progress=True, - skip_initial_state_check=True, + skip_initial_state_check=False, ): if sampler.iteration % auto_sample: continue @@ -502,6 +503,37 @@ def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sampl ] return autocorr +def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample=10, fractional_difference = 0.001): + autocorr = np.ones(shape=(iterations, n_params)) * np.nan + old_mean = np.inf + old_std = np.inf + for sample in sampler.sample( + start_points, + iterations=iterations, + progress=True, + skip_initial_state_check=False, + ): + if sampler.iteration % auto_sample: + continue + new_tau = sampler.get_autocorr_time(tol=0) + autocorr[sampler.iteration - 1] = new_tau + + dist_stats = stats.describe(sampler.get_chain(flat=True)) + + new_mean = dist_stats.mean + new_std = np.sqrt(dist_stats.variance) + + if all(np.abs(new_mean - old_mean) / old_mean < fractional_difference) and all(np.abs(new_std - old_std) / old_std < fractional_difference): + break + old_mean = new_mean + old_std = new_std + + autocorr = autocorr[ + : sampler.iteration, + ] + return autocorr + + def gelman_rubin(chain): ssq = np.var(chain, axis=1, ddof=1) w = np.mean(ssq, axis=0) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index fa6b06ea..88b8e6ec 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -120,6 +120,36 @@ "ts.ne", ] +FAST_OPT_QUANTITY = [ + # "xrcs.spectra", + "cxff_pi.ti", + "efit.wp", + "smmh1.ne", + "ts.te", + "ts.ne", +] + +FAST_OPT_PARAMS = [ + # "Ne_prof.y1", + "Ne_prof.y0", + # "Ne_prof.peaking", + # "Ne_prof.wcenter", + # "Ne_prof.wped", + # "Nimp_prof.y1", + # "Nimp_prof.y0", + # "Nimp_prof.wcenter", + # "Nimp_prof.wped", + # "Nimp_prof.peaking", + "Te_prof.y0", + # "Te_prof.wped", + # "Te_prof.wcenter", + # "Te_prof.peaking", + "Ti_prof.y0", + # "Ti_prof.wped", + # "Ti_prof.wcenter", + # "Ti_prof.peaking", +] + class BayesWorkflowExample(AbstractBayesWorkflow): def __init__( @@ -706,14 +736,14 @@ def __call__( run = BayesWorkflowExample( pulse=None, diagnostics=[ - "xrcs", + # "xrcs", "efit", "smmh1", "cxff_pi", "ts", ], - param_names=OPTIMISED_PARAMS, - opt_quantity=OPTIMISED_QUANTITY, + param_names=FAST_OPT_PARAMS, + opt_quantity=FAST_OPT_QUANTITY, priors=DEFAULT_PRIORS, profile_params=DEFAULT_PROFILE_PARAMS, phantoms=True, @@ -727,10 +757,12 @@ def __call__( tsample=0.05, ) run.setup_opt_data(phantoms=run.phantoms) - run.setup_optimiser(nwalkers=50, sample_method="high_density", model_kwargs=run.model_kwargs, nsamples=100) + + + run.setup_optimiser(nwalkers=20, sample_method="high_density", model_kwargs=run.model_kwargs, nsamples=50) # run.setup_optimiser(nwalkers=50, sample_method="ga", model_kwargs=run.model_kwargs, num_gens=50, popsize=100, sols_to_return=3, mutation_probability=None) results = run( - filepath=f"./results/test/", + filepath=f"./results/test_moments/", pulse_to_write=25000000, run="RUN01", mds_write=True, From 4730dc8985ea7217e21566af446b6baa113f41f4 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 12 Oct 2023 13:32:11 +0100 Subject: [PATCH 196/288] cleaned up and added virtual observations --- indica/bayesblackbox.py | 145 ++++++++++++++++++++ indica/bayesmodels.py | 267 ------------------------------------ indica/operators/gpr_fit.py | 227 +++++++++++------------------- 3 files changed, 225 insertions(+), 414 deletions(-) create mode 100644 indica/bayesblackbox.py delete mode 100644 indica/bayesmodels.py diff --git a/indica/bayesblackbox.py b/indica/bayesblackbox.py new file mode 100644 index 00000000..d2c15663 --- /dev/null +++ b/indica/bayesblackbox.py @@ -0,0 +1,145 @@ +from copy import deepcopy +import warnings + +import numpy as np +from scipy.stats import uniform + +np.seterr(all="ignore") +warnings.simplefilter("ignore", category=FutureWarning) + + +def gaussian(x, mean, sigma): + return 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-1 / 2 * ((x - mean) / sigma) ** 2) + + +def get_uniform(lower, upper): + # Less confusing parameterisation of scipy.stats uniform + return uniform(loc=lower, scale=upper - lower) + + +class BayesBlackBox: + """ + Bayesian black box model that creates _ln_posterior function from plasma and diagnostic model objects + + Parameters + ---------- + data + processed diagnostic data of format [diagnostic].[quantity] + plasma_context + plasma context has methods for using plasma object + model_handler + model_handler object to be called by ln_posterior + quant_to_optimise + quantity from data which will be optimised with bckc from diagnostic_models + priors + prior functions to apply to parameters e.g. scipy.stats.rv_continuous objects + + """ + + def __init__( + self, + data: dict, + quant_to_optimise: list, + priors: dict, + + plasma_context=None, + model_handler= None, + + percent_error: float = 0.10, + ): + self.data = data + self.quant_to_optimise = quant_to_optimise + self.priors = priors + + self.plasma_context = plasma_context + self.model_handler = model_handler + + self.percent_error = percent_error + + missing_data = list(set(quant_to_optimise).difference(data.keys())) + if missing_data: # gives list of keys in quant_to_optimise but not data + raise ValueError(f"{missing_data} not found in data given") + + def _ln_likelihood(self): + ln_likelihood = 0 + for key in self.quant_to_optimise: + time_coord = self.plasma_context.time_to_calculate + model_data = self.bckc[key] + exp_data = self.data[key].sel(t=time_coord) + exp_error = exp_data * self.percent_error + + if hasattr(self.data[key], "error"): + if (self.data[key].error != 0).any(): + # TODO: Some models have an error of 0 given + exp_error = self.data[key].error.sel(t=time_coord) + + _ln_likelihood = np.log(gaussian(model_data, exp_data, exp_error)) + # treat channel as key dim which isn't averaged like other dims + if "channel" in _ln_likelihood.dims: + _ln_likelihood = _ln_likelihood.sum(dim="channel", skipna=True) + ln_likelihood += _ln_likelihood.mean(skipna=True).values + return ln_likelihood + + def _ln_prior(self, parameters: dict): + ln_prior = 0 + for prior_name, prior_func in self.priors.items(): + param_names_in_prior = [x for x in parameters.keys() if x in prior_name] + if param_names_in_prior.__len__() == 0: + # if prior assigned but no parameter then skip + continue + param_values = [parameters[x] for x in param_names_in_prior] + if hasattr(prior_func, "pdf"): + # for scipy.stats objects use pdf / for lambda functions just call + ln_prior += np.log(prior_func.pdf(*param_values)) + else: + # if lambda prior with 2+ args is defined when only 1 of + # its parameters is given ignore it + if prior_func.__code__.co_argcount != param_values.__len__(): + continue + else: + # Sorting to make sure args are given in the same order + # as the prior_name string + name_index = [ + prior_name.find(param_name_in_prior) + for param_name_in_prior in param_names_in_prior + ] + sorted_name_index, sorted_param_values = ( + list(x) for x in zip(*sorted(zip(name_index, param_values))) + ) + ln_prior += np.log(prior_func(*sorted_param_values)) + return ln_prior + + + def ln_posterior(self, parameters: dict, **kwargs): + """ + Posterior probability given to optimisers + + Parameters + ---------- + parameters + inputs to optimise + kwargs + kwargs for models + + Returns + ------- + ln_posterior + log of posterior probability + blob + model outputs from bckc and kinetic profiles + """ + + ln_prior = self._ln_prior(parameters) + if ln_prior == -np.inf: # Don't call models if outside priors + return -np.inf, {} + + self.plasma_context.update_profiles(parameters) + plasma_attributes = self.plasma_context.return_plasma_attributes() + + self.bckc = self.model_handler._build_bckc(parameters, **kwargs) # model calls + + ln_likelihood = self._ln_likelihood() # compare results to data + ln_posterior = ln_likelihood + ln_prior + + blob = deepcopy({**self.bckc, **plasma_attributes}) + return ln_posterior, blob diff --git a/indica/bayesmodels.py b/indica/bayesmodels.py deleted file mode 100644 index b6b5f74b..00000000 --- a/indica/bayesmodels.py +++ /dev/null @@ -1,267 +0,0 @@ -from copy import deepcopy -import warnings - -import numpy as np -from scipy.stats import uniform - -np.seterr(all="ignore") -warnings.simplefilter("ignore", category=FutureWarning) - - -PLASMA_ATTRIBUTES = [ - "electron_temperature", - "electron_density", - "ion_temperature", - "ion_density", - "impurity_density", - "fast_density", - "neutral_density", - "zeff", - "wp", - "wth", - "ptot", - "pth", -] - - -def gaussian(x, mean, sigma): - return 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-1 / 2 * ((x - mean) / sigma) ** 2) - - -def get_uniform(lower, upper): - # Less confusing parameterisation of scipy.stats uniform - return uniform(loc=lower, scale=upper - lower) - - -class BayesModels: - """ - Object that is used with Plasma object to create ln_posterior - - Parameters - ---------- - plasma - Plasma object needed for the diagnostic model calls - data - processed diagnostic data of format [diagnostic].[quantity] - quant_to_optimise - quantity from data which will be optimised with bckc from diagnostic_models - priors - prior functions to apply to parameters e.g. scipy.stats.rv_continuous objects - diagnostic_models - model objects to be called by ln_posterior - """ - - def __init__( - self, - plasma=None, - data: dict = {}, - quant_to_optimise: list = [], - priors: dict = {}, - diagnostic_models: list = [], - percent_error: float = 0.10, - ): - self.plasma = plasma - self.data = data - self.quant_to_optimise = quant_to_optimise - self.diagnostic_models = diagnostic_models - self.priors = priors - self.percent_error = percent_error - - for diag_model in self.diagnostic_models: - diag_model.plasma = self.plasma - - missing_data = list(set(quant_to_optimise).difference(data.keys())) - if missing_data: # gives list of keys in quant_to_optimise but not data - raise ValueError(f"{missing_data} not found in data given") - - def _build_bckc(self, params, **kwargs): - """ - Parameters - ---------- - params - dictionary which is updated by optimiser - kwargs - passed to model i.e. settings - - Returns - ------- - bckc of results - """ - self.bckc: dict = {} - for model in self.diagnostic_models: - # removes "model.name." from params and kwargs then passes them to model - # e.g. xrcs.background -> background - _nuisance_params = { - param_name.replace(model.name + ".", ""): param_value - for param_name, param_value in params.items() - if model.name in param_name - } - _model_settings = { - kwarg_name.replace(model.name + ".", ""): kwarg_value - for kwarg_name, kwarg_value in kwargs.items() - if model.name in kwarg_name - } - - _model_kwargs = { - **_nuisance_params, - **_model_settings, - } # combine dictionaries - _bckc = model(**_model_kwargs) - _model_bckc = { - f"{model.name}.{value_name}": value - for value_name, value in _bckc.items() - } # prepend model name to bckc - self.bckc = dict(self.bckc, **_model_bckc) - return - - def _ln_likelihood(self): - ln_likelihood = 0 - for key in self.quant_to_optimise: - # Float128 since rounding of small numbers causes problems - # when initial results are bad fits - model_data = self.bckc[key].astype("float128") - exp_data = ( - self.data[key].sel(t=self.plasma.time_to_calculate).astype("float128") - ) - exp_error = ( - exp_data * self.percent_error - ) # Assume percentage error if none given. - if hasattr(self.data[key], "error"): - if ( - self.data[key].error != 0 - ).any(): # TODO: Some models have an error of 0 given - exp_error = self.data[key].error.sel( - t=self.plasma.time_to_calculate - ) - - _ln_likelihood = np.log(gaussian(model_data, exp_data, exp_error)) - # treat channel as key dim which isn't averaged like other dims - if "channel" in _ln_likelihood.dims: - _ln_likelihood = _ln_likelihood.sum(dim="channel", skipna=True) - ln_likelihood += _ln_likelihood.mean(skipna=True).values - return ln_likelihood - - def _ln_prior(self, parameters: dict): - ln_prior = 0 - for prior_name, prior_func in self.priors.items(): - param_names_in_prior = [x for x in parameters.keys() if x in prior_name] - if param_names_in_prior.__len__() == 0: - # if prior assigned but no parameter then skip - continue - param_values = [parameters[x] for x in param_names_in_prior] - if hasattr(prior_func, "pdf"): - # for scipy.stats objects use pdf / for lambda functions just call - ln_prior += np.log(prior_func.pdf(*param_values)) - else: - # if lambda prior with 2+ args is defined when only 1 of - # its parameters is given ignore it - if prior_func.__code__.co_argcount != param_values.__len__(): - continue - else: - # Sorting to make sure args are given in the same order - # as the prior_name string - name_index = [ - prior_name.find(param_name_in_prior) - for param_name_in_prior in param_names_in_prior - ] - sorted_name_index, sorted_param_values = ( - list(x) for x in zip(*sorted(zip(name_index, param_values))) - ) - ln_prior += np.log(prior_func(*sorted_param_values)) - return ln_prior - - def sample_from_priors(self, param_names, size=10): - # Use priors to generate samples - for name in param_names: - if name in self.priors.keys(): - if hasattr(self.priors[name], "rvs"): - continue - else: - raise TypeError(f"prior object {name} missing rvs method") - else: - raise ValueError(f"Missing prior for {name}") - - # Throw out samples that don't meet conditional priors and redraw - samples = np.empty((param_names.__len__(), 0)) - while samples.size < param_names.__len__() * size: - # Some mangling of dictionaries so _ln_prior works - # Increase size * n if too slow / looping too much - new_sample = { - name: self.priors[name].rvs(size=size * 2) for name in param_names - } - ln_prior = self._ln_prior(new_sample) - # Convert from dictionary of arrays -> array, - # then filtering out where ln_prior is -infinity - accepted_samples = np.array(list(new_sample.values()))[ - :, ln_prior != -np.inf - ] - samples = np.append(samples, accepted_samples, axis=1) - samples = samples[:, 0:size] - return samples.transpose() - - def sample_from_high_density_region( - self, param_names: list, sampler, nwalkers: int, nsamples=100 - ): - # TODO: implement smarter MLE (maximum likelihood estimate) - start_points = self.sample_from_priors(param_names, size=nsamples) - - ln_prob, _ = sampler.compute_log_prob(start_points) - num_best_points = 3 - index_best_start = np.argsort(ln_prob)[-num_best_points:] - best_start_points = start_points[index_best_start, :] - best_points_std = np.std(best_start_points, axis=0) - - # Passing samples through ln_prior and redrawing if they fail - samples = np.empty((param_names.__len__(), 0)) - while samples.size < param_names.__len__() * nwalkers: - sample = np.random.normal( - np.mean(best_start_points, axis=0), - best_points_std, - size=(nwalkers * 5, len(param_names)), - ) - start = {name: sample[:, idx] for idx, name in enumerate(param_names)} - ln_prior = self._ln_prior(start) - # Convert from dictionary of arrays -> array, - # then filtering out where ln_prior is -infinity - accepted_samples = np.array(list(start.values()))[:, ln_prior != -np.inf] - samples = np.append(samples, accepted_samples, axis=1) - start_points = samples[:, 0:nwalkers].transpose() - return start_points - - def ln_posterior(self, parameters: dict, **kwargs): - """ - Posterior probability given to optimisers - - Parameters - ---------- - parameters - inputs to optimise - kwargs - kwargs for models - - Returns - ------- - ln_posterior - log of posterior probability - blob - model outputs from bckc and kinetic profiles - """ - - ln_prior = self._ln_prior(parameters) - if ln_prior == -np.inf: # Don't call models if outside priors - return -np.inf, {} - - self.plasma.update_profiles(parameters) - self._build_bckc(parameters, **kwargs) # model calls - ln_likelihood = self._ln_likelihood() # compare results to data - ln_posterior = ln_likelihood + ln_prior - - plasma_attributes = {} - for plasma_key in PLASMA_ATTRIBUTES: - if hasattr(self.plasma, plasma_key): - plasma_attributes[plasma_key] = getattr(self.plasma, plasma_key).sel( - t=self.plasma.time_to_calculate - ) - else: - raise ValueError(f"plasma does not have attribute {plasma_key}") - - blob = deepcopy({**self.bckc, **plasma_attributes}) - return ln_posterior, blob diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index 0659c744..112c8e5a 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -32,8 +32,8 @@ def choose_kernel(kernel: str): """ kernels = { - "RBF_noise": RadialBasisFunction_NoNoise(), - "RBF": RadialBasisFunction_WithNoise(), + "RBF_noise": RadialBasisFunction_WithNoise(), + "RBF": RadialBasisFunction_NoNoise(), } if kernel in kernels.keys(): @@ -42,7 +42,7 @@ def choose_kernel(kernel: str): raise ValueError -def RadialBasisFunction_NoNoise(length_scale=0.1, dlength_scale=0.001, **kwargs): +def RadialBasisFunction_NoNoise(length_scale=0.2, dlength_scale=0.001, **kwargs): kernel = 1.0 * kernels.RBF( length_scale=length_scale, length_scale_bounds=( @@ -75,8 +75,6 @@ def gpr_fit( y_err: np.array, x_fit: np.array, kernel_name: str = "RBF_noise", - y_bounds: tuple = (1, 1), - err_bounds: tuple = (0, 0), ): """ Run GPR fit given input data and desired x-grid @@ -93,10 +91,6 @@ def gpr_fit( x-axis for fitting kernel Kernel name - y_bounds - Boundery value at the edge of the x-grid - err_bounds - Boundery error at the edge of the x-grid Returns ------- @@ -107,14 +101,9 @@ def gpr_fit( kernel = choose_kernel(kernel_name) isort = np.argsort(x) - _x = np.sort(np.append(x, [x_fit[0], x_fit[-1]])) + _x = np.sort(x) _y = np.interp(_x, x[isort], y[isort]) - - _y[0] = y_bounds[0] - _y[-1] = y_bounds[1] _y_err = np.interp(_x, x[isort], y_err[isort]) - _y_err[0] = err_bounds[0] - _y_err[-1] = err_bounds[1] idx = np.isfinite(_y) @@ -122,8 +111,8 @@ def gpr_fit( _y_err = _y_err[idx] _y = _y[idx].reshape(-1, 1) - gpr = GaussianProcessRegressor(kernel=kernel, alpha=_y_err**2) # alpha is sigma^2 - gpr.fit(_x, _y) + gpr = GaussianProcessRegressor(kernel=kernel, alpha=_y_err**2,) # alpha is sigma^2 + gpr.fit(_x, _y, ) _x_fit = x_fit.reshape(-1, 1) y_fit, y_fit_err = gpr.predict(_x_fit, return_std=True) @@ -131,192 +120,134 @@ def gpr_fit( return y_fit, y_fit_err -def run_gpr_fit( - data: DataArray, - xdim: str = "R", - kernel_name: str = "RBF_noise", - x_bounds: tuple = (0, 1), - y_bounds: tuple = (1, 1), - err_bounds: tuple = (0, 0), -): - """ - Run GPR fit for experimental Indica-native DataArray structures - - Parameters - ---------- - data - Data to be fitted dims = (x, t) - xdim - Dimension on which fit is to be performed - kernel_name - Name of kernel to be used for fitting - - Returns - ------- - - """ - - x_fit = np.linspace(x_bounds[0], x_bounds[1], 1000) - - y_fit = [] - y_fit_err = [] - for t in data.t: - x = data.R.values - y = data.sel(t=t).values - y_err = data.error.sel(t=t).values - _y_fit, _y_fit_err = gpr_fit( - x, - y, - y_err, - x_fit, - kernel_name=kernel_name, - y_bounds=y_bounds, - err_bounds=err_bounds, - ) - - y_fit.append(DataArray(_y_fit, coords=[(xdim, x_fit)])) - y_fit_err.append(DataArray(_y_fit_err, coords=[(xdim, x_fit)])) - - fit = xr.concat(y_fit, "t").assign_coords(t=data.t) - fit_err = xr.concat(y_fit_err, "t").assign_coords(t=data.t) - - return fit, fit_err - - def plot_gpr_fit( data: DataArray, - fit: DataArray, - fit_err: DataArray, - tplot: float, - x_data: DataArray = None, - x_fit: DataArray = None, + y_fit: DataArray, + y_fit_err: DataArray, fig_style: str = "profiles", ylabel: str = "", xlabel: str = "", label: str = "", title: str = "", fig_name: str = "", - figure: bool = True, save_fig: bool = False, color=None, ): set_plot_rcparams(fig_style) - if figure: - plt.figure() + plt.figure() - if x_data is None: - xdim = [dim for dim in data.dims if dim != "t"][0] - x_data = getattr(data, xdim) - - if x_fit is None: - xdim = [dim for dim in fit.dims if dim != "t"][0] - x_fit = getattr(fit, xdim) - - if "t" in x_data.dims: - _x_data = x_data.sel(t=tplot) - else: - _x_data = x_data - - if "t" in x_fit.dims: - _x_fit = x_fit.sel(t=tplot) - else: - _x_fit = x_fit - - y_data = data.sel(t=tplot) - y_err = data.error.sel(t=tplot) - - y_fit = fit.sel(t=tplot) - y_fit_err = fit_err.sel(t=tplot) + x_data = getattr(data, data.dims[0]) + x_fit = getattr(y_fit, data.dims[0]) + y_err = data.error.sel(t=data.t) plt.fill_between( - _x_fit, + x_fit, y_fit - y_fit_err, y_fit + y_fit_err, alpha=0.5, color=color, ) - plt.plot(_x_fit, y_fit, color=color) - plt.errorbar(_x_data, y_data, y_err, linestyle="", color=color) - plt.plot(_x_data, y_data, marker="o", linestyle="", color=color, label=label) + plt.plot(x_fit, y_fit, color=color) + plt.errorbar(x_data, data, y_err, linestyle="", color=color) + plt.plot(x_data, data, marker="o", linestyle="", color=color, label=label) plt.xlabel(xlabel) plt.ylabel(ylabel) - plt.title(f"{title} t = {tplot:.3f} s") + plt.title(f"{title} t = {data.t.values:.3f} s") set_axis_sci() if len(label) > 0: plt.legend() if save_fig: - save_figure(FIG_PATH, f"{fig_name}_GPR_fit_{tplot:.3f}_s", save_fig=save_fig) + save_figure(FIG_PATH, f"{fig_name}_GPR_fit_{data.t:.3f}_s", save_fig=save_fig) def example_run( pulse: int = 10619, - tstart=0.02, + tstart=0.06, tend=0.1, kernel_name: str = "RBF_noise", plot=True, save_fig=False, xdim: str = "R", - x_bounds: tuple = (0, 1), - y_bounds: tuple = (1, 1), - err_bounds: tuple = (0, 0), + split="LFS", + virtual_obs=True, + ): st40 = ReadST40(pulse, tstart, tend) st40(instruments=["ts", "efit"]) + rmag = st40.binned_data["efit"]["rmag"] data = st40.raw_data["ts"]["te"] - if xdim not in data.dims and hasattr(data, xdim): - data = data.swap_dims({"channel": xdim}) + data.transform.set_equilibrium(st40.equilibrium) + data.transform.convert_to_rho_theta(t=data.t) + data["rho"] = data.transform.rho + if xdim == "R": x_bounds = data.transform._machine_dims[0] - if hasattr(data, "equilibrium"): - data.transform.convert_to_rho_theta(t=data.t) - - fit, fit_err = run_gpr_fit( - data, - kernel_name=kernel_name, - x_bounds=x_bounds, - y_bounds=y_bounds, - err_bounds=err_bounds, - xdim=xdim, - ) + else: + x_bounds = (0, 1.2) + + data = data[(data.t >= tstart) & (data.t <= tend)] + rmag = rmag[0] + + if split == "HFS": + data = data.where(data.R < rmag) + elif split == "LFS": + data = data.where(data.R > rmag) + else: + data = data + + x_fit = np.linspace(x_bounds[0], x_bounds[1], 1000) + dx = x_fit[1] - x_fit[0] + y_fit = [] + y_fit_err = [] + for t in data.t: + if "t" in data.__getattr__(xdim).dims: + x = data.__getattr__(xdim).sel(t=t).values + else: + x = data.__getattr__(xdim).values + y = data.sel(t=t).values + y_err = data.error.sel(t=t).values + + if virtual_obs: + x = np.insert(x, [0, x.size], [x_bounds[0], x_bounds[1]]) + if xdim == "rho": + y = np.insert(y, [0, y.size], [np.nanmax(y), 0]) + else: + y = np.insert(y, [0, y.size], [0, 0]) + + y_err = np.insert(y_err, [0, y_err.size], [1, 1]) + + _y_fit, _y_fit_err = gpr_fit( + x, + y, + y_err, + x_fit, + kernel_name=kernel_name, + ) + y_fit.append(DataArray(_y_fit, coords=[(xdim, x_fit)])) + y_fit_err.append(DataArray(_y_fit_err, coords=[(xdim, x_fit)])) + + fit = xr.concat(y_fit, "t").assign_coords(t=data.t) + fit_err = xr.concat(y_fit_err, "t").assign_coords(t=data.t) if plot or save_fig: plt.ioff() fig_name = f"{pulse}_TS_Te" for tplot in data.t.values: plot_gpr_fit( - data, - fit, - fit_err, - tplot, + data.sel(t=tplot).swap_dims({"channel":xdim}), + fit.sel(t=tplot), + fit_err.sel(t=tplot), ylabel="Te [eV]", - xlabel="R [m]", + xlabel=f"{xdim}", title=str(st40.pulse), fig_name=f"{fig_name}_vs_R", save_fig=save_fig, ) - if hasattr(data, "equilibrium"): - x_data = data.transform.rho.assign_coords( - R=("channel", data.R) - ).swap_dims({"channel": "R"}) - x_fit = x_data.interp(R=fit.R) - plot_gpr_fit( - data, - fit, - fit_err, - tplot, - x_data=x_data, - x_fit=x_fit, - ylabel="Te [eV]", - xlabel="rho-poloidal", - title=str(st40.pulse), - fig_name=f"{fig_name}_vs_rho", - save_fig=save_fig, - ) if not save_fig: plt.show() @@ -324,4 +255,6 @@ def example_run( if __name__ == "__main__": - example_run() + + # example_run(pulse=11314, xdim="rho", split="HFS", virtual_obs=True, kernel_name="RBF") + example_run(pulse=11314, xdim="rho", split="LFS", virtual_obs=True, kernel_name="RBF") From ec77188a1a1d90b09810d1760df60799c7f10306 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 12 Oct 2023 14:05:46 +0100 Subject: [PATCH 197/288] normalising TS profiles before fitting --- indica/operators/gpr_fit.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index 112c8e5a..40c15e92 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -54,7 +54,7 @@ def RadialBasisFunction_NoNoise(length_scale=0.2, dlength_scale=0.001, **kwargs) def RadialBasisFunction_WithNoise( - length_scale=0.1, dlength_scale=0.001, noise_level=1000, dnoise_level=3000, **kwargs + length_scale=0.2, dlength_scale=0.001, noise_level=10, dnoise_level=10, **kwargs ): kernel = 1.0 * kernels.RBF( length_scale=length_scale, @@ -139,7 +139,7 @@ def plot_gpr_fit( x_data = getattr(data, data.dims[0]) x_fit = getattr(y_fit, data.dims[0]) - y_err = data.error.sel(t=data.t) + y_err = data.error plt.fill_between( x_fit, @@ -172,6 +172,7 @@ def example_run( xdim: str = "R", split="LFS", virtual_obs=True, + quant = "te", ): @@ -179,11 +180,21 @@ def example_run( st40(instruments=["ts", "efit"]) rmag = st40.binned_data["efit"]["rmag"] - data = st40.raw_data["ts"]["te"] + data = st40.raw_data["ts"][quant] data.transform.set_equilibrium(st40.equilibrium) data.transform.convert_to_rho_theta(t=data.t) data["rho"] = data.transform.rho + # Normalising + if quant == "ne": + data.values = data.values * 1e-19 + data["error"] = data.error * 1e-19 + units = "n19" + else: + data.values = data.values * 1e-3 + data["error"] = data.error * 1e-3 + units = "keV" + if xdim == "R": x_bounds = data.transform._machine_dims[0] else: @@ -200,7 +211,7 @@ def example_run( data = data x_fit = np.linspace(x_bounds[0], x_bounds[1], 1000) - dx = x_fit[1] - x_fit[0] + # dx = x_fit[1] - x_fit[0] y_fit = [] y_fit_err = [] for t in data.t: @@ -218,7 +229,7 @@ def example_run( else: y = np.insert(y, [0, y.size], [0, 0]) - y_err = np.insert(y_err, [0, y_err.size], [1, 1]) + y_err = np.insert(y_err, [0, y_err.size], [0.01, 0.01]) _y_fit, _y_fit_err = gpr_fit( x, @@ -235,13 +246,13 @@ def example_run( if plot or save_fig: plt.ioff() - fig_name = f"{pulse}_TS_Te" + fig_name = f"{pulse}_TS_{quant}" for tplot in data.t.values: plot_gpr_fit( data.sel(t=tplot).swap_dims({"channel":xdim}), fit.sel(t=tplot), fit_err.sel(t=tplot), - ylabel="Te [eV]", + ylabel=f"{quant} ({units})", xlabel=f"{xdim}", title=str(st40.pulse), fig_name=f"{fig_name}_vs_R", @@ -256,5 +267,7 @@ def example_run( if __name__ == "__main__": - # example_run(pulse=11314, xdim="rho", split="HFS", virtual_obs=True, kernel_name="RBF") - example_run(pulse=11314, xdim="rho", split="LFS", virtual_obs=True, kernel_name="RBF") + # example_run(pulse=11314, xdim="rho", split="HFS", virtual_obs=True, kernel_name="RBF_noise", quant="ne") + example_run(pulse=11314, xdim="rho", split="LFS", virtual_obs=True, kernel_name="RBF_noise", quant="ne") + # example_run(pulse=11314, xdim="rho", split="HFS", virtual_obs=True, kernel_name="RBF_noise", quant="te") + # example_run(pulse=11314, xdim="rho", split="LFS", virtual_obs=True, kernel_name="RBF_noise", quant="te") From fd8e640bbc145e869f9ac07a845bc815dbcc84cc Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 12 Oct 2023 16:30:51 +0100 Subject: [PATCH 198/288] background can be class attribute or call argument --- indica/models/helike_spectroscopy.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index dcf50731..7e3fc5db 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -43,6 +43,7 @@ def __init__( window: np.array = None, window_masks=None, line_labels=None, + background=None, ): """ Read all atomic data and initialise objects @@ -71,6 +72,7 @@ def __init__( self.window_masks = window_masks self.line_ranges = LINE_RANGES self.line_labels = line_labels + self.background = background if window is None: window = np.linspace(window_lim[0], window_lim[1], window_len) @@ -417,22 +419,22 @@ def __call__( if hasattr(self, "plasma"): if t is None: t = self.plasma.time_to_calculate - Te = self.plasma.electron_temperature.interp( + Te = self.plasma.electron_temperature.sel( t=t, ) - Ne = self.plasma.electron_density.interp( + Ne = self.plasma.electron_density.sel( t=t, ) - Nh = self.plasma.neutral_density.interp( + Nh = self.plasma.neutral_density.sel( t=t, ) Fz = {} _Fz = self.plasma.fz for elem in _Fz.keys(): - Fz[elem] = _Fz[elem].interp(t=t) + Fz[elem] = _Fz[elem].sel(t=t) - Ti = self.plasma.ion_temperature.interp(t=t) - Nimp = self.plasma.impurity_density.interp(t=t) + Ti = self.plasma.ion_temperature.sel(t=t) + Nimp = self.plasma.impurity_density.sel(t=t) else: if ( Ne is None @@ -444,6 +446,10 @@ def __call__( ): raise ValueError("Give inputs or assign plasma class!") + if background is None: + if self.background is not None: + background = self.background.sel(t=t) + self.t = t self.Te = Te self.Ne = Ne From 3196e03efbff61acb9a7271e6bdf0c0236112999 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 12 Oct 2023 16:31:21 +0100 Subject: [PATCH 199/288] placeholder for filtering methods --- indica/readers/read_st40.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/indica/readers/read_st40.py b/indica/readers/read_st40.py index f14f999e..8b2c305e 100644 --- a/indica/readers/read_st40.py +++ b/indica/readers/read_st40.py @@ -298,6 +298,7 @@ def __call__( self, instruments: list = None, revisions: dict = None, + filters: dict = None, map_raw: bool = False, tstart: float = None, tend: float = None, @@ -311,8 +312,12 @@ def __call__( self.debug = debug if instruments is None: instruments = list(REVISIONS.keys()) - if revisions is None: + if not revisions: + # TODO: fix default behaviour if missing key revisions = REVISIONS + if not filters: + # TODO: fix default behaviour if missing key + filters = FILTER_LIMITS if tstart is None: tstart = self.tstart if tend is None: From 4979c2ac8c1fae44721f09f17a0c323a3c4942d5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 12 Oct 2023 16:32:13 +0100 Subject: [PATCH 200/288] gutted workflow class / methods removed to context objects --- indica/workflows/abstract_bayes_workflow.py | 224 +------------------- 1 file changed, 1 insertion(+), 223 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 20faa75c..27ec77dd 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -28,109 +28,7 @@ def __init__( self.param_names = param_names self.opt_quantity = opt_quantity self.priors = priors - self.read_data(self.diagnostics) - self.setup_models(self.diagnostics) - def read_test_data( - self, diagnostic_transforms: dict, tstart=None, tend=None, dt=None - ): - # Used with phantom data for purposes of tests - print("Reading fake data") - self.equilibrium = fake_equilibrium( - tstart, - tend, - dt, - ) - self.transforms = diagnostic_transforms - self.data: dict = {} - - def read_data(self, diagnostics: list, tstart=None, tend=None, dt=None): - self.reader = ReadST40(self.pulse, tstart=tstart, tend=tend, dt=dt) - self.reader(diagnostics) - - missing_keys = set(diagnostics) - set(self.reader.binned_data.keys()) - if len(missing_keys) > 0: - raise ValueError(f"missing data: {missing_keys}") - - self.equilibrium = self.reader.equilibrium - self.transforms = self.reader.transforms - self.data = self.reader.binned_data - - @abstractmethod - def setup_plasma(self): - """ - Contains all methods and settings for setting up / initialising plasma object - """ - self.plasma = None - self.plasma.set_equilibrium(self.reader.equilibrium) - self.save_phantom_profiles() - - @abstractmethod - def setup_models(self, diagnostics: list): - """ - Initialising models normally requires data to be read so transforms can be set - - """ - self.models: dict = {} - - @abstractmethod - def _phantom_data(self): - opt_data = {} - return opt_data - - @abstractmethod - def _exp_data(self): - opt_data = {} - return opt_data - - @abstractmethod - def setup_opt_data(self, phantom: bool = False): - """ - Get and prepare the data in necessary format for optimiser - """ - for model in self.models: - model.plasma = self.plasma - - if phantom: - self.opt_data = self._phantom_data() - else: - self.opt_data = self._exp_data() - - @abstractmethod - def setup_optimiser(self, model_kwargs): - """ - Initialise and provide settings for optimiser - """ - self.bayesopt = None - - def save_phantom_profiles(self, kinetic_profiles=None): - if kinetic_profiles is None: - kinetic_profiles = [ - "electron_density", - "impurity_density", - "electron_temperature", - "ion_temperature", - "ion_density", - "fast_density", - "neutral_density", - ] - if self.phantoms: - phantom_profiles = { - profile_key: getattr(self.plasma, profile_key) - .sel(t=self.plasma.time_to_calculate) - .copy() - for profile_key in kinetic_profiles - } - else: - phantom_profiles = { - profile_key: getattr(self.plasma, profile_key).sel( - t=self.plasma.time_to_calculate - ) - * 0 - for profile_key in kinetic_profiles - } - - self.phantom_profiles = phantom_profiles def _build_inputs_dict(self): """ @@ -332,7 +230,7 @@ def _build_result_dict(self, ): "PRIOR_SAMPLE": self.prior_sample, "POST_SAMPLE": self.post_sample, "AUTOCORR": self.autocorr, - "GELMANRUBIN": gelman_rubin(self.sampler.get_chain(flat=False)) + # "GELMANRUBIN": gelman_rubin(self.sampler.get_chain(flat=False)) } result["GLOBAL"] = { @@ -409,47 +307,6 @@ def _build_result_dict(self, ): } return result - def run_sampler(self, iterations, burn_frac): - """ - TODO: unsure if keeping in abstract class is best practice - - Runs the sampler and saves certain attributes from the sampler - - Returns - - result in MDSPlus node formatting - - """ - self.burn_frac = burn_frac - self.iterations = iterations - - self.autocorr = sample_with_moments( - self.sampler, - self.start_points, - iterations, - self.param_names.__len__(), - auto_sample=10, - fractional_difference=0.05 - ) - _blobs = self.sampler.get_blobs(discard=int(self.sampler.iteration * burn_frac), flat=True) - blobs = [blob for blob in _blobs if blob] # remove empty blobs - - blob_names = blobs[0].keys() - self.samples = np.arange(0, blobs.__len__()) - - - self.blobs = { - blob_name: xr.concat( - [data[blob_name] for data in blobs], - dim=pd.Index(self.samples, name="index"), - ) - for blob_name in blob_names - } - self.accept_frac = self.sampler.acceptance_fraction.sum() - self.prior_sample = self.bayesmodel.sample_from_priors( - self.param_names, size=int(1e4) - ) - self.post_sample = self.sampler.get_chain(discard=int(self.sampler.iteration * burn_frac), flat=True) def save_pickle(self, result, filepath): if filepath: @@ -457,17 +314,6 @@ def save_pickle(self, result, filepath): with open(filepath + "results.pkl", "wb") as handle: pickle.dump(result, handle) - def dict_of_dataarray_to_numpy(self, dict_of_dataarray): - """ - Mutates input dictionary to change xr.DataArray objects to np.array - - """ - for key, value in dict_of_dataarray.items(): - if isinstance(value, dict): - self.dict_of_dataarray_to_numpy(value) - elif isinstance(value, xr.DataArray): - dict_of_dataarray[key] = dict_of_dataarray[key].values - return dict_of_dataarray @abstractmethod def __call__(self, filepath="./results/test/", **kwargs): @@ -477,71 +323,3 @@ def __call__(self, filepath="./results/test/", **kwargs): return self.result - -def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sample=5,): - autocorr = np.ones(shape=(iterations, n_params)) * np.nan - old_tau = np.inf - for sample in sampler.sample( - start_points, - iterations=iterations, - progress=True, - skip_initial_state_check=False, - ): - if sampler.iteration % auto_sample: - continue - new_tau = sampler.get_autocorr_time(tol=0) - autocorr[ - sampler.iteration - 1, - ] = new_tau - converged = np.all(new_tau * 50 < sampler.iteration) - converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) - if converged: - break - old_tau = new_tau - autocorr = autocorr[ - : sampler.iteration, - ] - return autocorr - -def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample=10, fractional_difference = 0.001): - autocorr = np.ones(shape=(iterations, n_params)) * np.nan - old_mean = np.inf - old_std = np.inf - for sample in sampler.sample( - start_points, - iterations=iterations, - progress=True, - skip_initial_state_check=False, - ): - if sampler.iteration % auto_sample: - continue - new_tau = sampler.get_autocorr_time(tol=0) - autocorr[sampler.iteration - 1] = new_tau - - dist_stats = stats.describe(sampler.get_chain(flat=True)) - - new_mean = dist_stats.mean - new_std = np.sqrt(dist_stats.variance) - - if all(np.abs(new_mean - old_mean) / old_mean < fractional_difference) and all(np.abs(new_std - old_std) / old_std < fractional_difference): - break - old_mean = new_mean - old_std = new_std - - autocorr = autocorr[ - : sampler.iteration, - ] - return autocorr - - -def gelman_rubin(chain): - ssq = np.var(chain, axis=1, ddof=1) - w = np.mean(ssq, axis=0) - theta_b = np.mean(chain, axis=1) - theta_bb = np.mean(theta_b, axis=0) - m = chain.shape[0] - n = chain.shape[1] - B = n/(m-1) * np.sum((theta_bb - theta_b)**2, axis=0) - var_theta = (n-1) / n * w + 1 / n * B - R = np.sqrt(var_theta / w) - return R From 97bcfa8bcc533ad512bf6b254f474a068b85c40c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 12 Oct 2023 16:32:32 +0100 Subject: [PATCH 201/288] gutted workflow class / methods removed to context objects --- indica/workflows/bayes_workflow_example.py | 1307 ++++++++++++-------- 1 file changed, 789 insertions(+), 518 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index 88b8e6ec..c14608a4 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -1,14 +1,19 @@ -from typing import Any, List, Tuple +from typing import Any, List, Tuple, Callable from typing import Dict +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass, field +from functools import partial import emcee import flatdict import numpy as np -from scipy.stats import loguniform +from scipy.stats import loguniform, describe import xarray as xr +import pandas as pd -from indica.bayesmodels import BayesModels -from indica.bayesmodels import get_uniform +from indica.bayesblackbox import BayesBlackBox, ln_prior +from indica.bayesblackbox import get_uniform from indica.models.charge_exchange import ChargeExchange from indica.models.charge_exchange import pi_transform_example from indica.models.equilibrium_reconstruction import EquilibriumReconstruction @@ -25,8 +30,8 @@ from indica.writers.bda_tree import write_nodes from indica.readers.read_st40 import ReadST40 from indica.equilibrium import Equilibrium - - +from indica.equilibrium import fake_equilibrium +from indica.readers.read_st40 import ReadST40 # global configurations DEFAULT_PROFILE_PARAMS = { @@ -120,6 +125,22 @@ "ts.ne", ] +DEFAULT_DIAG_NAMES = [ + "xrcs", + "efit", + "smmh1", + "cxff_pi", + "ts", +] + +FAST_DIAG_NAMES = [ + # "xrcs", + "efit", + "smmh1", + "cxff_pi", + "ts", +] + FAST_OPT_QUANTITY = [ # "xrcs.spectra", "cxff_pi.ti", @@ -151,106 +172,129 @@ ] -class BayesWorkflowExample(AbstractBayesWorkflow): - def __init__( - self, - diagnostics: list, - param_names: list, - opt_quantity: list, - priors: dict, - profile_params: dict, - pulse: int = None, - tstart=0.02, - tend=0.10, - dt=0.005, - - phantoms: bool = False, - fast_particles = False, - astra_run=None, - astra_pulse_range=13000000, - astra_equilibrium=False, - efit_revision = 0, - set_ts_profiles = False, - set_all_profiles=False, - astra_wp = False, +def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample=10, fractional_difference=0.001): + autocorr = np.ones(shape=(iterations, n_params)) * np.nan + old_mean = np.inf + old_std = np.inf + for sample in sampler.sample( + start_points, + iterations=iterations, + progress=True, + skip_initial_state_check=False, ): - self.pulse = pulse - self.diagnostics = diagnostics - self.param_names = param_names - self.opt_quantity = opt_quantity - self.priors = priors - self.profile_params = profile_params - self.tstart = tstart - self.tend = tend - self.dt = dt - - self.phantoms = phantoms - self.fast_particles = fast_particles - self.astra_run= astra_run - self.astra_pulse_range = astra_pulse_range - self.astra_equilibrium = astra_equilibrium - self.efit_revision = efit_revision - self.set_ts_profiles = set_ts_profiles - self.set_all_profiles = set_all_profiles - self.astra_wp = astra_wp - - self.model_kwargs = {} - - for attribute in [ - "param_names", - "opt_quantity", - "priors", - "diagnostics", - "profile_params", - ]: - if getattr(self, attribute) is None: - raise ValueError(f"{attribute} needs to be defined") - - if self.pulse is None and self.phantoms is False: - raise ValueError( - "Set phantoms to True when running phantom plasma i.e. pulse=None" - ) - - # TODO: Add some abstraction here - if pulse is None: - print("Running in test mode") - example_transforms = { - "xrcs": helike_transform_example(1), - "smmh1": smmh1_transform_example(1), - "cxff_pi": pi_transform_example(5), - "ts":ts_transform_example(11), - } - self.read_test_data( - example_transforms, tstart=self.tstart, tend=self.tend, dt=self.dt - ) - else: - self.read_data( - self.diagnostics, tstart=self.tstart, tend=self.tend, dt=self.dt - ) - if self.efit_revision != 0: - efit_equilibrium = self.reader.reader_equil.get("", "efit", self.efit_revision) - self.equilibrium = Equilibrium(efit_equilibrium) - - self.setup_models(self.diagnostics) - - def setup_plasma( - self, - tstart=None, - tend=None, - dt=None, - tsample=None, - main_ion="h", - impurities=("ar", "c"), - impurity_concentration=(0.001, 0.04), - n_rad=20, - **kwargs, + if sampler.iteration % auto_sample: + continue + new_tau = sampler.get_autocorr_time(tol=0) + autocorr[sampler.iteration - 1] = new_tau + + dist_stats = describe(sampler.get_chain(flat=True)) + + new_mean = dist_stats.mean + new_std = np.sqrt(dist_stats.variance) + + if all(np.abs(new_mean - old_mean) / old_mean < fractional_difference) and all( + np.abs(new_std - old_std) / old_std < fractional_difference): + break + old_mean = new_mean + old_std = new_std + + autocorr = autocorr[ + : sampler.iteration, + ] + return autocorr + + +def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sample=5, ): + autocorr = np.ones(shape=(iterations, n_params)) * np.nan + old_tau = np.inf + for sample in sampler.sample( + start_points, + iterations=iterations, + progress=True, + skip_initial_state_check=False, + ): + if sampler.iteration % auto_sample: + continue + new_tau = sampler.get_autocorr_time(tol=0) + autocorr[ + sampler.iteration - 1, + ] = new_tau + converged = np.all(new_tau * 50 < sampler.iteration) + converged &= np.all(np.abs(old_tau - new_tau) / new_tau < 0.01) + if converged: + break + old_tau = new_tau + autocorr = autocorr[ + : sampler.iteration, + ] + return autocorr + + +def gelman_rubin(chain): + ssq = np.var(chain, axis=1, ddof=1) + w = np.mean(ssq, axis=0) + theta_b = np.mean(chain, axis=1) + theta_bb = np.mean(theta_b, axis=0) + m = chain.shape[0] + n = chain.shape[1] + B = n / (m - 1) * np.sum((theta_bb - theta_b) ** 2, axis=0) + var_theta = (n - 1) / n * w + 1 / n * B + R = np.sqrt(var_theta / w) + return R + + +def dict_of_dataarray_to_numpy(self, dict_of_dataarray): + """ + Mutates input dictionary to change xr.DataArray objects to np.array + + """ + for key, value in dict_of_dataarray.items(): + if isinstance(value, dict): + self.dict_of_dataarray_to_numpy(value) + elif isinstance(value, xr.DataArray): + dict_of_dataarray[key] = dict_of_dataarray[key].values + return dict_of_dataarray + + +@dataclass +class BayesSettings: + diagnostics: list = field(default_factory=lambda: DEFAULT_DIAG_NAMES) + param_names: list = field(default_factory=lambda: OPTIMISED_PARAMS) + opt_quantity: list = field(default_factory=lambda: OPTIMISED_QUANTITY) + priors: dict = field(default_factory=lambda: DEFAULT_PRIORS) + + """ + TODO: default methods / getter + setters + print warning if using default values + """ + + def __post_init__(self): + missing_quantities = [quant for quant in self.opt_quantity if quant.split(".")[0] not in self.diagnostics] + if missing_quantities: + raise ValueError(f"{missing_quantities} missing the relevant diagnostic") + + +@dataclass +class PlasmaContext: + equilibrium: Equilibrium = None + profile_params: dict = field(default_factory=lambda: DEFAULT_PROFILE_PARAMS) + + """ + set profiles / profiler + """ + + def init_plasma( + self, + equilibrium: Equilibrium, + tstart=None, + tend=None, + dt=None, + main_ion="h", + impurities=("ar", "c"), + impurity_concentration=(0.001, 0.04), + n_rad=20, ): - if not all([tstart, tend, dt]): - tstart = self.tstart - tend = self.tend - dt = self.dt - # TODO: move to plasma.py self.plasma = Plasma( tstart=tstart, tend=tend, @@ -262,449 +306,634 @@ def setup_plasma( n_rad=n_rad, ) - if tsample == None: - self.tsample = self.plasma.t - else: - self.tsample = self.plasma.t[ - np.abs(tsample - self.plasma.t).argmin() + self.plasma.set_equilibrium(equilibrium) + self.update_profiles(self.profile_params) + self.plasma.build_atomic_data(calc_power_loss=False) + + def update_profiles(self, params: dict): + if not hasattr(self, "plasma"): + raise ValueError("plasma not initialised") + + self.plasma.update_profiles(params) + + def time_iterator(self): # TODO: Why is this being called anytime plasma attributes are accessed? + print("resetting time iterator") + return iter(self.plasma.t) + + def return_plasma_attrs(self): + PLASMA_ATTRIBUTES = [ + "electron_temperature", + "electron_density", + "ion_temperature", + "ion_density", + "impurity_density", + "fast_density", + "neutral_density", + "zeff", + "wp", + "wth", + "ptot", + "pth", + ] + plasma_attributes = {} + for plasma_key in PLASMA_ATTRIBUTES: + if hasattr(self.plasma, plasma_key): + plasma_attributes[plasma_key] = getattr(self.plasma, plasma_key).sel( + t=self.plasma.time_to_calculate + ) + else: + raise ValueError(f"plasma does not have attribute {plasma_key}") + return plasma_attributes + + def save_phantom_profiles(self, kinetic_profiles=None, phantoms=None): + if kinetic_profiles is None: + kinetic_profiles = [ + "electron_density", + "impurity_density", + "electron_temperature", + "ion_temperature", + "ion_density", + "fast_density", + "neutral_density", ] + if phantoms: + phantom_profiles = { + profile_key: getattr(self.plasma, profile_key) + .sel(t=self.plasma.time_to_calculate) + .copy() + for profile_key in kinetic_profiles + } + else: + phantom_profiles = { + profile_key: getattr(self.plasma, profile_key).sel( + t=self.plasma.time_to_calculate + ) + * 0 + for profile_key in kinetic_profiles + } + self.phantom_profiles = phantom_profiles + + # + # + # + # def _init_fast_particles(self, run="RUN602", ): + # + # st40_code = ReadST40(self.astra_pulse_range + self.pulse, self.tstart-self.dt, self.tend+self.dt, dt=self.dt, tree="astra") + # astra_data = st40_code.get_raw_data("", "astra", run) + # + # if self.astra_equilibrium: + # self.equilibrium = Equilibrium(astra_data) + # self.plasma.equilibrium = self.equilibrium + # + # st40_code.bin_data_in_time(["astra"], self.tstart, self.tend, self.dt) + # code_data = st40_code.binned_data["astra"] + # Nf = ( + # code_data["nf"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + # * 1.0e19 + # ) + # self.plasma.fast_density.values = Nf.values + # Nn = ( + # code_data["nn"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + # * 1.0e19 + # ) + # self.plasma.neutral_density.values = Nn.values + # Pblon = code_data["pblon"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + # self.plasma.pressure_fast_parallel.values = Pblon.values + # Pbper = code_data["pbper"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) + # self.plasma.pressure_fast_perpendicular.values = Pbper.values + # self.astra_data = code_data + # + # if self.set_ts_profiles: + # overwritten_params = [param for param in self.param_names if any(xs in param for xs in ["Te", "Ne"])] + # if any(overwritten_params): + # raise ValueError(f"Te/Ne set by TS but then the following params overwritten: {overwritten_params}") + # Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 + # self.plasma.Te_prof = lambda: Te.values + # Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 + # self.plasma.Ne_prof = lambda: Ne.values + # + # if self.set_all_profiles: + # overwritten_params = [param for param in self.param_names if any(xs in param for xs in ["Te", "Ti", "Ne", "Nimp"])] + # if any(overwritten_params): + # raise ValueError(f"Te/Ne set by TS but then the following params overwritten: {overwritten_params}") + # Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 + # self.plasma.Te_prof = lambda: Te.values + # Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 + # self.plasma.Ne_prof = lambda: Ne.values + # Ti = code_data["ti"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 + # self.plasma.Ti_prof = lambda: Ti.values + # Nimp = code_data["niz1"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 + # self.plasma.Nimp_prof = lambda: Nimp.values + + +@dataclass +class ModelContext: + diagnostics: list + plasma: Plasma + equilibrium: Equilibrium + transforms: dict + model_kwargs: Dict[str, Dict[str, Any]] + + """ + Setup models so that they have transforms / plasma / equilibrium etc.. everything needed to produce bckc from models + + TODO: remove repeating code / likely make general methods + """ + + def update_model_kwargs(self, data): + if self.model_kwargs is None: + self.model_kwargs = {} + for diag in diagnostics: + if diag not in self.model_kwargs.keys(): + self.model_kwargs[diag] = {} - self.plasma.time_to_calculate = self.tsample - self.plasma.set_equilibrium(self.equilibrium) - self.plasma.update_profiles(self.profile_params) - if self.fast_particles: - self._init_fast_particles(run=self.astra_run) - self.plasma.update_profiles({}) + if "xrcs" in data.keys(): + self.model_kwargs["xrcs"]["window"] = data["xrcs"]["spectra"].wavelength.values - self.plasma.build_atomic_data(calc_power_loss=False) - self.save_phantom_profiles() + # TODO: handling model calls dependent on exp data + background = data["xrcs"]["spectra"].where( + (data["xrcs"]["spectra"].wavelength < 0.392) + & (data["xrcs"]["spectra"].wavelength > 0.388), + drop=True, + ) + self.model_kwargs["xrcs"]["background"] = background.mean(dim="wavelength") + + def init_models(self, ): + if not hasattr(self, "plasma"): + raise ValueError("needs plasma to setup_models") - def setup_models(self, diagnostics: list): self.models: Dict[str, Any] = {} - model: Any = None - for diag in diagnostics: + for diag in self.diagnostics: if diag == "smmh1": - los_transform = self.transforms[diag] - # machine_dims = ((0.15, 0.95), (-0.7, 0.7)) - # origin = np.array([[-0.38063365, 0.91893092, 0.01]]) - # # end = np.array([[0, 0, 0.01]]) - # direction = np.array([[0.38173721, -0.92387953, -0.02689453]]) - # los_transform = LineOfSightTransform( - # origin[:, 0], - # origin[:, 1], - # origin[:, 2], - # direction[:, 0], - # direction[:, 1], - # direction[:, 2], - # name="", - # machine_dimensions=machine_dims, - # passes=2, - # ) - los_transform.set_equilibrium(self.equilibrium, force=True) - model = Interferometry(name=diag) - model.set_los_transform(los_transform) - - elif diag == "xrcs": - los_transform = self.transforms[diag] - los_transform.set_equilibrium(self.equilibrium, force=True) - window = None - if hasattr(self, "data"): - if diag in self.data.keys(): - window = self.data[diag]["spectra"].wavelength.values - - model = HelikeSpectrometer( - name="xrcs", - window_masks=[slice(0.394, 0.396)], - window=window, - ) - model.set_los_transform(los_transform) + self.transforms[diag].set_equilibrium(self.equilibrium, force=True) + self.models[diag] = Interferometry(name=diag, **self.model_kwargs[diag]) + self.models[diag].set_los_transform(self.transforms[diag]) elif diag == "efit": - model = EquilibriumReconstruction(name="efit") + self.models[diag] = EquilibriumReconstruction(name=diag, **self.model_kwargs[diag]) elif diag == "cxff_pi": - transform = self.transforms[diag] - transform.set_equilibrium(self.equilibrium, force=True) - model = ChargeExchange(name=diag, element="ar") - model.set_transect_transform(transform) + self.transforms[diag].set_equilibrium(self.equilibrium, force=True) + self.models[diag] = ChargeExchange(name=diag, **self.model_kwargs[diag]) + self.models[diag].set_transect_transform(self.transforms[diag]) elif diag == "cxff_tws_c": - transform = self.transforms[diag] - transform.set_equilibrium(self.equilibrium, force=True) - model = ChargeExchange(name=diag, element="c") - model.set_transect_transform(transform) + self.transforms[diag].set_equilibrium(self.equilibrium, force=True) + self.models[diag] = ChargeExchange(name=diag, **self.model_kwargs[diag]) + self.models[diag].set_transect_transform(self.transforms[diag]) elif diag == "ts": - transform = self.transforms[diag] - transform.set_equilibrium(self.equilibrium, force=True) - model = ThomsonScattering(name=diag, ) - model.set_transect_transform(transform) - else: - raise ValueError(f"{diag} not found in setup_models") - self.models[diag] = model - - def _init_fast_particles(self, run="RUN602", ): + self.transforms[diag].set_equilibrium(self.equilibrium, force=True) + self.models[diag] = ThomsonScattering(name=diag, **self.model_kwargs[diag]) + self.models[diag].set_transect_transform(self.transforms[diag]) - st40_code = ReadST40(self.astra_pulse_range + self.pulse, self.tstart-self.dt, self.tend+self.dt, dt=self.dt, tree="astra") - astra_data = st40_code.get_raw_data("", "astra", run) - - if self.astra_equilibrium: - self.equilibrium = Equilibrium(astra_data) - self.plasma.equilibrium = self.equilibrium + elif diag == "xrcs": + self.transforms[diag].set_equilibrium(self.equilibrium, force=True) + self.models[diag] = HelikeSpectrometer(name="xrcs", **self.model_kwargs[diag]) + self.models[diag].set_los_transform(self.transforms[diag]) + else: + raise ValueError(f"{diag} not implemented in ModelHandler.setup_models") - st40_code.bin_data_in_time(["astra"], self.tstart, self.tend, self.dt) - code_data = st40_code.binned_data["astra"] - Nf = ( - code_data["nf"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - * 1.0e19 - ) - self.plasma.fast_density.values = Nf.values - Nn = ( - code_data["nn"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - * 1.0e19 - ) - self.plasma.neutral_density.values = Nn.values - Pblon = code_data["pblon"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - self.plasma.pressure_fast_parallel.values = Pblon.values - Pbper = code_data["pbper"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - self.plasma.pressure_fast_perpendicular.values = Pbper.values - self.astra_data = code_data - - if self.set_ts_profiles: - overwritten_params = [param for param in self.param_names if any(xs in param for xs in ["Te", "Ne"])] - if any(overwritten_params): - raise ValueError(f"Te/Ne set by TS but then the following params overwritten: {overwritten_params}") - Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 - self.plasma.Te_prof = lambda: Te.values - Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 - self.plasma.Ne_prof = lambda: Ne.values - - if self.set_all_profiles: - overwritten_params = [param for param in self.param_names if any(xs in param for xs in ["Te", "Ti", "Ne", "Nimp"])] - if any(overwritten_params): - raise ValueError(f"Te/Ne set by TS but then the following params overwritten: {overwritten_params}") - Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 - self.plasma.Te_prof = lambda: Te.values - Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 - self.plasma.Ne_prof = lambda: Ne.values - Ti = code_data["ti"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 - self.plasma.Ti_prof = lambda: Ti.values - Nimp = code_data["niz1"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 - self.plasma.Nimp_prof = lambda: Nimp.values - - def setup_opt_data(self, phantoms=False, **kwargs): - if not hasattr(self, "plasma"): - raise ValueError("Missing plasma object required for setup_opt_data") - for model in self.models.values(): # Maybe refactor here... + for model_name, model in self.models.items(): model.plasma = self.plasma - if phantoms: - self.opt_data = self._phantom_data(**kwargs) - else: - self.opt_data = self._exp_data(**kwargs) - - def _phantom_data(self, noise=False, noise_factor=0.1, **kwargs): - opt_data = {} - if "smmh1" in self.diagnostics: - opt_data["smmh1.ne"] = ( - self.models["smmh1"]() - .pop("ne") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - ) - if "xrcs" in self.diagnostics: - opt_data["xrcs.spectra"] = ( - self.models["xrcs"]() - .pop("spectra") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - ) - opt_data["xrcs.spectra"]["error"] = np.sqrt(opt_data["xrcs.spectra"]) - if "cxff_pi" in self.diagnostics: - cxrs_data = ( - self.models["cxff_pi"]() - .pop("ti") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - ) - opt_data["cxff_pi.ti"] = cxrs_data.where(cxrs_data.channel == 2, drop=True) - - if "ts" in self.diagnostics: - _ts_data = self.models["ts"]() - ts_data = {key: _ts_data[key].expand_dims(dim={"t": [self.plasma.time_to_calculate]}) for key in ["te", "ne"]} - opt_data["ts.te"] = ts_data["te"] - opt_data["ts.ne"] = ts_data["ne"] - opt_data["ts.te"]["error"] = opt_data["ts.te"] / opt_data["ts.te"] * ( - 0.10 * opt_data["ts.te"].max(dim="channel")) - opt_data["ts.ne"]["error"] = opt_data["ts.ne"] / opt_data["ts.ne"] * ( - 0.10 * opt_data["ts.ne"].max(dim="channel")) - - if "efit" in self.diagnostics: - opt_data["efit.wp"] = ( - self.models["efit"]() - .pop("wp") - .expand_dims(dim={"t": [self.plasma.time_to_calculate]}) - ) + return self.models - # TODO: add chers - - if noise: - #TODO: add TS - opt_data["smmh1.ne"] = opt_data["smmh1.ne"] + opt_data[ - "smmh1.ne" - ].max().values * np.random.normal(0, noise_factor, None) - opt_data["xrcs.spectra"] = opt_data["xrcs.spectra"] + np.random.normal( - 0, - np.sqrt( - opt_data["xrcs.spectra"].values[ - 0, - ] - ), - opt_data["xrcs.spectra"].shape[1], - ) - opt_data["cxff_pi.ti"] = opt_data["cxff_pi.ti"] + opt_data[ - "cxff_pi.ti" - ].max().values * np.random.normal( - 0, noise_factor, opt_data["cxff_pi.ti"].shape[1] - ) - opt_data["efit.wp"] = opt_data["efit.wp"] + opt_data[ - "efit.wp" - ].max().values * np.random.normal(0, noise_factor, None) + def _build_bckc(self, params: dict, **kwargs): + """ + Parameters + ---------- + params - dictionary which is updated by optimiser + kwargs - passed to model call i.e. settings + + Returns + ------- + bckc of results + """ - return opt_data + # Float128 since rounding of small numbers causes problems + # when initial results are bad fits + # model_data = self.bckc[key].astype("float128") + + self.bckc: dict = {} + for model_name, model in self.models.items(): + # removes "model.name." from params and kwargs then passes them to model + # e.g. xrcs.background -> background + _nuisance_params = { + param_name.replace(model.name + ".", ""): param_value + for param_name, param_value in params.items() + if model.name in param_name + } + _model_settings = { + kwarg_name.replace(model.name + ".", ""): kwarg_value + for kwarg_name, kwarg_value in kwargs.items() + if model.name in kwarg_name + } - def _exp_data(self, **kwargs): - opt_data = flatdict.FlatDict(self.data, ".") - if "xrcs" in self.diagnostics: - opt_data["xrcs.spectra"]["wavelength"] = ( - opt_data["xrcs.spectra"].wavelength - ) + _model_kwargs = { + **_nuisance_params, + **_model_settings, + } # combine dictionaries + _bckc = model(**_model_kwargs) + _model_bckc = { + f"{model.name}.{value_name}": value + for value_name, value in _bckc.items() + } # prepend model name to bckc + self.bckc = dict(self.bckc, **_model_bckc) + return self.bckc + + +@dataclass +class ReaderSettings: + revisions: dict = field(default_factory=lambda: {}) + filters: dict = field(default_factory=lambda: {}) + + +@dataclass +class DataContext(ABC): + reader_settings: ReaderSettings + tstart: float + tend: float + dt: float + transforms = None + equilibrium = None + + @abstractmethod + def read_data(self, ): + self.equilbrium = None + self.transforms = None + self.raw_data = None + self.binned_data = None + + def check_if_data_present(self, data_strategy: Callable): + if not self.binned_data: + print("Data not given: using data strategy") + self.binned_data = data_strategy() + + @abstractmethod + def process_data(self, model_context: ModelContext): + self.model_context = model_context # TODO: handle this dependency (phantom data) some other way + self.check_if_data_present(lambda: {}) + self.opt_data = flatdict.FlatDict(self.binned_data) + + +@dataclass +class ExpData(DataContext): + pulse: int + diagnostics: list + + """ + Considering: either rewriting this class to take over from ReadST40 or vice versa + + """ + + def read_data(self, ): + self.reader = ReadST40(self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt, ) + self.reader(self.diagnostics, revisions=self.reader_settings.revisions, filters=self.reader_settings.filters) + missing_keys = set(diagnostics) - set(self.reader.binned_data.keys()) + if len(missing_keys) > 0: + raise ValueError(f"missing data: {missing_keys}") + self.equilibrium = self.reader.equilibrium + self.transforms = self.reader.transforms + # TODO raw data included + self.raw_data = self.reader.raw_data + self.binned_data = self.reader.binned_data + + def process_data(self, model_callables: Callable): + self.model_callables = model_callables + self.check_if_data_present(self._fail_strategy) + self.opt_data = self._process_data() + + def _fail_strategy(self): + raise ValueError("Data strategy: Fail") + + def _process_data(self): + opt_data = flatdict.FlatDict(self.binned_data, ".") + + if "xrcs.spectra" in opt_data.keys(): background = opt_data["xrcs.spectra"].where( (opt_data["xrcs.spectra"].wavelength < 0.392) & (opt_data["xrcs.spectra"].wavelength > 0.388), drop=True, ) - self.model_kwargs["xrcs.background"] = background.mean( - dim="wavelength" - ).sel(t=self.plasma.time_to_calculate) opt_data["xrcs.spectra"]["error"] = np.sqrt( opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 ) - - if "cxff_pi" in self.diagnostics: - opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( - opt_data["cxff_pi"]["ti"] != 0, - ) - - opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( - opt_data["cxff_pi"]["ti"].channel > 3, - ) - opt_data["cxff_pi"]["ti"] = opt_data["cxff_pi"]["ti"].where( - opt_data["cxff_pi"]["ti"].channel < 6, - ) - - if "cxff_tws_c" in self.diagnostics: - opt_data["cxff_tws_c"]["ti"] = opt_data["cxff_tws_c"]["ti"].where( - opt_data["cxff_tws_c"]["ti"] != 0, - ) - opt_data["cxff_tws_c"]["ti"] = opt_data["cxff_tws_c"]["ti"].where( - opt_data["cxff_tws_c"]["ti"].channel < 2, - ) - - if "ts" in self.diagnostics: - # TODO: fix error, for now flat error - opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel >19, drop=True) - opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel > 19, drop=True) - - opt_data["ts.te"]["error"] = opt_data["ts.te"].max(dim="channel") * 0.05 + if "ts.ne" in opt_data.keys(): opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 - if self.astra_wp: - opt_data["efit.wp"] = self.astra_data["wth"] + if "ts.te" in opt_data.keys(): + opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 return opt_data - def setup_optimiser( - self, - model_kwargs, - nwalkers=50, - burn_frac=0.10, - sample_method="random", - **kwargs, - ): - self.model_kwargs = model_kwargs - self.nwalkers = nwalkers - self.burn_frac = burn_frac - self.sample_method = sample_method - - self.bayesmodel = BayesModels( - plasma=self.plasma, - data=self.opt_data, - diagnostic_models=[*self.models.values()], - quant_to_optimise=self.opt_quantity, - priors=self.priors, - ) - ndim = len(self.param_names) +@dataclass +class MockData(DataContext): + diagnostic_transforms: dict = field(default_factory=lambda: {}) + model_call_params: dict = field(default_factory=lambda: {}) + + def read_data(self): + # Used with phantom data for purposes of tests + print("Reading mock equilibrium / transforms") + self.equilibrium = fake_equilibrium( + tstart, + tend, + dt, + ) + self.transforms = self.diagnostic_transforms + self.binned_data: dict = {} + self.raw_data: dict = {} + + + def process_data(self, model_callables: Callable): + self.model_callables = model_callables + self.check_if_data_present(self._gen_data_strategy) + + def _gen_data_strategy(self): + print("Data strategy: generating from model bckc") + self.binned_data = self._process_data() + + def _process_data(self): + binned_data = self.model_callables() + self.opt_data = flatdict.FlatDict(binned_data, ".") + return binned_data + + +@dataclass +class PhantomData(DataContext): + pulse: int + diagnostics: list + model_call_kwargs: dict = field(default_factory=lambda: {}) + + def read_data(self, ): + self.reader = ReadST40(self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt, ) + self.reader(self.diagnostics, revisions=self.reader_settings.revisions, filters=self.reader_settings.filters) + missing_keys = set(diagnostics) - set(self.reader.binned_data.keys()) + if len(missing_keys) > 0: + raise ValueError(f"missing data: {missing_keys}") + self.equilibrium = self.reader.equilibrium + self.transforms = self.reader.transforms + self.raw_data = {} + self.binned_data = {} + + def process_data(self, model_callables: Callable): + self.model_callables = model_callables + self.check_if_data_present(self._gen_data_strategy) + + def _gen_data_strategy(self): + print("Data strategy: generating from model bckc") + self.binned_data = self._process_data() + + def _process_data(self): + binned_data = self.model_callables() + self.opt_data = flatdict.FlatDict(binned_data, ".") + return binned_data + + +@dataclass +class OptimiserEmceeSettings: + param_names: list + priors: dict + iterations: int = 1000 + nwalkers: int = 50 + burn_frac: float = 0.20 + sample_method: str = "random" + starting_samples: int = 100 + stopping_criterion: str = "mode" + stopping_criterion_fract: float = 0.05 + + +@dataclass +class OptimiserContext(ABC): + optimiser_settings: OptimiserEmceeSettings + + @abstractmethod + def init_optimiser(self, *args, **kwargs): + self.optimiser = None + + @abstractmethod + def sample_start_points(self, *args, **kwargs): + self.start_points = None + + @abstractmethod + def format_results(self): + self.results = {} + + @abstractmethod + def run(self): + results = None + return results + + +@dataclass +class EmceeOptimiser(OptimiserContext): + + def init_optimiser(self, blackbox_func: Callable, model_kwargs: dict = field(default={}) + ): + ndim = len(self.optimiser_settings.param_names) self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] - self.sampler = emcee.EnsembleSampler( - nwalkers, + self.optimiser = emcee.EnsembleSampler( + self.optimiser_settings.nwalkers, ndim, - log_prob_fn=self.bayesmodel.ln_posterior, - parameter_names=self.param_names, + log_prob_fn=blackbox_func, + parameter_names=self.optimiser_settings.param_names, moves=self.move, kwargs=model_kwargs, ) + def sample_start_points(self, ): + if self.optimiser_settings.sample_method == "high_density": - def _sample_start_points(self, sample_method: str = "random", nsamples=100, **kwargs): - if sample_method == "high_density": - start_points = self.bayesmodel.sample_from_high_density_region( - self.param_names, self.sampler, self.nwalkers, nsamples=nsamples + self.start_points = self.sample_from_high_density_region( + param_names=self.optimiser_settings.param_names, + priors=self.optimiser_settings.priors, + optimiser=self.optimiser, + nwalkers=self.optimiser_settings.nwalkers, + nsamples=self.optimiser_settings.starting_samples ) - elif sample_method == "ga": - self.start_points = self._sample_start_points(sample_method="random", **kwargs) - samples_in_weird_format = self.ga_opt(**kwargs) - sample_start_points = np.array([idx[1] for idx in samples_in_weird_format]) + elif self.optimiser_settings.sample_method == "random": + self.start_points = self.sample_from_priors( + param_names=self.optimiser_settings.param_names, + priors=self.optimiser_settings.priors, + size=self.optimiser_settings.nwalkers + ) + else: + raise ValueError(f"Sample method: {self.optimiser_settings.sample_method} " + f"not recognised, Defaulting to random sampling") + + def sample_from_priors(self, param_names: list, priors: dict, size=10): + # Use priors to generate samples + for name in param_names: + if name in priors.keys(): + if hasattr(priors[name], "rvs"): + continue + else: + raise TypeError(f"prior object {name} missing rvs method") + else: + raise ValueError(f"Missing prior for {name}") + + # Throw out samples that don't meet conditional priors and redraw + samples = np.empty((param_names.__len__(), 0)) + while samples.size < param_names.__len__() * size: + # Some mangling of dictionaries so _ln_prior works + # Increase size * n if too slow / looping too much + new_sample = { + name: priors[name].rvs(size=size * 2) for name in param_names + } + _ln_prior = ln_prior(priors, new_sample) + # Convert from dictionary of arrays -> array, + # then filtering out where ln_prior is -infinity + accepted_samples = np.array(list(new_sample.values()))[ + :, _ln_prior != -np.inf + ] + samples = np.append(samples, accepted_samples, axis=1) + samples = samples[:, 0:size] + return samples.transpose() + + def sample_from_high_density_region( + self, param_names: list, priors: dict, optimiser, nwalkers: int, nsamples=100 + ): - start_points = np.random.normal( - np.mean(sample_start_points, axis=0), - np.std(sample_start_points, axis=0), - size=(self.nwalkers, sample_start_points.shape[1]), + # TODO: remove repeated code + start_points = self.sample_from_priors(param_names, priors, size=nsamples) + + ln_prob, _ = optimiser.compute_log_prob(start_points) + num_best_points = 3 + index_best_start = np.argsort(ln_prob)[-num_best_points:] + best_start_points = start_points[index_best_start, :] + best_points_std = np.std(best_start_points, axis=0) + + # Passing samples through ln_prior and redrawing if they fail + samples = np.empty((param_names.__len__(), 0)) + while samples.size < param_names.__len__() * nwalkers: + sample = np.random.normal( + np.mean(best_start_points, axis=0), + best_points_std, + size=(nwalkers * 5, len(param_names)), ) + start = {name: sample[:, idx] for idx, name in enumerate(param_names)} + _ln_prior = ln_prior(priors, start, ) + # Convert from dictionary of arrays -> array, + # then filtering out where ln_prior is -infinity + accepted_samples = np.array(list(start.values()))[:, _ln_prior != -np.inf] + samples = np.append(samples, accepted_samples, axis=1) + start_points = samples[:, 0:nwalkers].transpose() + return start_points + def run(self, ): - elif sample_method == "random": - start_points = self.bayesmodel.sample_from_priors( - self.param_names, size=self.nwalkers + if self.optimiser_settings.stopping_criterion == "mode": + self.autocorr = sample_with_moments( + self.optimiser, + self.start_points, + self.optimiser_settings.iterations, + self.optimiser_settings.param_names.__len__(), + auto_sample=10, + fractional_difference=self.optimiser_settings.stopping_criterion_fract ) - else: - print(f"Sample method: {sample_method} not recognised, Defaulting to random sampling") - start_points = self.bayesmodel.sample_from_priors( - self.param_names, size=self.nwalkers - ) - return start_points + raise ValueError(f"Stopping criterion: {self.optimiser_settings.stopping_criterion} not recognised") + optimiser_results = self.format_results() + return optimiser_results - def ga_opt(self, num_gens=30, popsize=50, sols_to_return=5, mutation_probability=None, **kwargs) -> list(tuple((float, []))): - """Runs the GA optimization, and returns a number of the best solutions. Uses - a population convergence stopping criteria: fitness does not improve in 3 successive generations, we stop. + def format_results(self): + results = {} + _blobs = self.optimiser.get_blobs(discard=int(self.optimiser.iteration * self.optimiser_settings.burn_frac), + flat=True) + blobs = [blob for blob in _blobs if blob] # remove empty blobs - Args: - num_gens (int, optional): Maximum number of generations to run. Defaults to 30. - popsize (int, optional): Population size. Defaults to 50. - sols_to_return (int, optional): How many of the best solutions the function shall return. Defaults to 5. + blob_names = blobs[0].keys() + samples = np.arange(0, blobs.__len__()) - Returns: - list(tuple(float, np.Array(float))): list of tuples, where first element is fitness, second np.array of the parameters. - """ + results["blobs"] = { + blob_name: xr.concat( + [data[blob_name] for data in blobs], + dim=pd.Index(samples, name="index"), + ) + for blob_name in blob_names + } + results["accept_frac"] = self.optimiser.acceptance_fraction.sum() + results["prior_sample"] = self.sample_from_priors( + self.optimiser_settings.param_names, self.optimiser_settings.priors, size=int(1e4) + ) + results["post_sample"] = self.optimiser.get_chain( + discard=int(self.optimiser.iteration * self.optimiser_settings.burn_frac), flat=True) + return results - import pygad - import time - - # Packaged evaluation function - def idiot_proof(ga_instance, x, sol_idx): - res, _ = self.bayesmodel.ln_posterior(dict(zip(self.param_names, x))) - # print(-res) - return float(res) - - print(f"Running GA for a maximum of {num_gens} generations of {popsize} individuals each.") - - # Initialize the GA instance - ga_instance = pygad.GA(num_generations=num_gens, - num_parents_mating=20, - sol_per_pop=popsize, - num_genes=len(self.start_points[0]), - fitness_func=idiot_proof, - initial_population=self.start_points, - save_best_solutions=True, - stop_criteria="saturate_5", - mutation_probability=mutation_probability) - - st = time.time() - # Execute - ga_instance.run() - - print( - f"Time ran: {time.time() - st:.2f} seconds. Ran total of {ga_instance.generations_completed} generations.") - - # Saves the fitness evolution plot - # figure = ga_instance.plot_fitness() - # figure.savefig(f'GA_plot.png', dpi=300) - - # Organizing all the non-inf individuals from the last generation - feasible_indices = [i for i in range(len(ga_instance.last_generation_fitness)) if - ga_instance.last_generation_fitness[i] != -np.inf] - feasible_individuals = [ga_instance.population[i] for i in feasible_indices] - feasible_fitnesses = [ga_instance.last_generation_fitness[i] for i in feasible_indices] - - # for i, item in enumerate(feasible_individuals_with_keywords): - # item["fitness"]=feasible_fitnesses[i - # feasible_individuals_with_keywords=sorted(feasible_individuals_with_keywords,key= lambda d:d['fitness'],reverse=True) - # feasible_individuals_and_fitnesses=[tuple(feasible_fitnesses[i],feasible_individuals[i]) for i in len(feasible_individuals)] - - # Combining the last individuals to a collection and sorting - feasible_individuals_and_fitnesses = [] - for i in range(len(feasible_fitnesses)): - feasible_individuals_and_fitnesses.append(tuple((feasible_fitnesses[i], feasible_individuals[i]))) - feasible_individuals_and_fitnesses = sorted(feasible_individuals_and_fitnesses, key=lambda x: x[0], - reverse=True) - - return feasible_individuals_and_fitnesses[:sols_to_return] - def __call__( - self, - filepath="./results/test/", - run=None, - mds_write=False, - pulse_to_write=None, - plot=False, - iterations=100, - burn_frac=0.10, - **kwargs, +class BayesWorkflowExample(AbstractBayesWorkflow): + def __init__( + self, + bayes_settings: BayesSettings, + data_context: DataContext, + plasma_context: PlasmaContext, + model_context: ModelContext, + optimiser_context: EmceeOptimiser, + + tstart: float = 0.02, + tend: float = 0.10, + dt: float = 0.01, + ): + self.bayes_settings = bayes_settings + self.data_context = data_context + self.plasma_context = plasma_context + self.model_context = model_context + self.optimiser_context = optimiser_context - self.iterations = iterations - self.burn_frac = burn_frac + self.tstart = tstart + self.tend = tend + self.dt = dt - if mds_write: - # check_analysis_run(self.pulse, self.run) - self.node_structure = create_nodes( - pulse_to_write=pulse_to_write, - run=run, - diagnostic_quantities=self.opt_quantity, - mode="EDIT", - ) + self.plasma_context.init_plasma(equilibrium=data_context.equilibrium, tstart=tstart, tend=tend, dt=dt, ) + self.plasma_context.save_phantom_profiles(phantoms=False) - self.result = self._build_inputs_dict() - results = [] - if not self.tsample.shape: - self.tsample = np.array([self.tsample]) - self.plasma.time_to_calculate = self.tsample[0] - self.start_points = self._sample_start_points( - sample_method=self.sample_method, **kwargs - ) + self.model_context.update_model_kwargs(data_context.data) + self.model_context.init_models() - for t in self.tsample: - self.plasma.time_to_calculate = t - print(f"Time: {t}") - self.run_sampler(iterations=iterations, burn_frac=burn_frac) - _result = self._build_result_dict() - results.append(_result) + self.model_call_kwargs = {"xrcs.pixel_offset": 4.0} + self.data_context.process_data(partial(self.model_context._build_bckc, self.model_call_kwargs)) - self.start_points = self.sampler.get_chain()[-1,:,:] - self.sampler.reset() + self.blackbox = BayesBlackBox(data=self.data_context.opt_data, + plasma_context=self.plasma_context, + model_context=self.model_context, + quant_to_optimise=self.bayes_settings.opt_quantity, + priors=self.bayes_settings.priors) - _result = dict(_result, ** self.result) + self.optimiser_context.init_optimiser(self.blackbox.ln_posterior, model_kwargs={}) - self.save_pickle(_result, filepath=filepath, ) + def __call__( + self, + filepath="./results/test/", + run=None, + mds_write=False, + pulse_to_write=None, + plot=False, + **kwargs, + ): - if plot: # currently requires result with DataArrays - plot_bayes_result(_result, filepath) + self.result = self._build_inputs_dict() + + results = [] + + for time in self.plasma_context.time_iterator(): + self.plasma_context.plasma.time_to_calculate = time + print(f"Time: {self.plasma_context.plasma.time_to_calculate}") + self.optimiser_context.sample_start_points() + self.optimiser_context.run() + results.append(self.optimiser_context.format_results()) + self.optimiser_context.optimiser.reset() + + # result = self._build_result_dict() + + # Combine the blobs/results from optimiser and add time dim - self.result = dict(self.result, ** results[-1]) profiles = {} globals = {} for key, prof in results[0]["PROFILES"].items(): @@ -716,57 +945,99 @@ def __call__( _profs = [result["PROFILES"][key] for result in results] profiles[key] = xr.concat(_profs, self.tsample) - for key, prof in results[0]["GLOBAL"].items(): _glob = [result["GLOBAL"][key] for result in results] globals[key] = xr.concat(_glob, self.tsample) - result = {"PROFILES":profiles, "GLOBAL":globals} + result = {"PROFILES": profiles, "GLOBAL": globals} - self.result = dict(self.result, **result,) + self.result = dict(self.result, **result, ) + _result = dict(result, **self.result) self.result = self.dict_of_dataarray_to_numpy(self.result) + + self.save_pickle(_result, filepath=filepath, ) + + if plot: # currently requires result with DataArrays + plot_bayes_result(_result, filepath) + if mds_write: + # check_analysis_run(self.pulse, self.run) + self.node_structure = create_nodes( + pulse_to_write=pulse_to_write, + run=run, + diagnostic_quantities=self.bayes_settings.opt_quantity, + mode="EDIT", + ) write_nodes(pulse_to_write, self.node_structure, self.result) - return self.result + return if __name__ == "__main__": - run = BayesWorkflowExample( - pulse=None, - diagnostics=[ - # "xrcs", - "efit", - "smmh1", - "cxff_pi", - "ts", - ], - param_names=FAST_OPT_PARAMS, - opt_quantity=FAST_OPT_QUANTITY, - priors=DEFAULT_PRIORS, - profile_params=DEFAULT_PROFILE_PARAMS, - phantoms=True, - fast_particles=False, - tstart=0.02, - tend=0.10, - dt=0.005, - ) - - run.setup_plasma( - tsample=0.05, - ) - run.setup_opt_data(phantoms=run.phantoms) - - - run.setup_optimiser(nwalkers=20, sample_method="high_density", model_kwargs=run.model_kwargs, nsamples=50) - # run.setup_optimiser(nwalkers=50, sample_method="ga", model_kwargs=run.model_kwargs, num_gens=50, popsize=100, sols_to_return=3, mutation_probability=None) - results = run( - filepath=f"./results/test_moments/", - pulse_to_write=25000000, - run="RUN01", - mds_write=True, - plot=True, - burn_frac=0.2, - iterations=500, - ) + # mock_transforms = {"xrcs": helike_transform_example(1), + # "smmh1": smmh1_transform_example(1), + # "cxff_pi": pi_transform_example(5), + # "ts": ts_transform_example(11), } + + pulse = 11216 + tstart = 0.05 + tend = 0.06 + dt = 0.01 + + diagnostics = [ + "xrcs", + "efit", + # "smmh1", + # "cxff_pi", + # "ts", + ] + opt_quant = ["xrcs.spectra", "efit.wp"] + opt_params = ["Te_prof.y0", + "Ti_prof.y0", + "Ne_prof.y0"] + + bayes_settings = BayesSettings(diagnostics=diagnostics, param_names=opt_params, + opt_quantity=opt_quant, priors=DEFAULT_PRIORS, ) + + data_settings = ReaderSettings(filters={}, + revisions={}) # Add general methods for filtering data co-ords to ReadST40 + + data_context = ExpData(pulse=pulse, diagnostics=diagnostics, + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + data_context.read_data() + + plasma_settings = PlasmaSettings() + plasma_context = PlasmaContext(plasma_settings = plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) + + + + model_init_kwargs = {"cxff_pi": {"element": "ar"}, + "cxff_tws_c": {"element": "c"}, + "xrcs": { + "window_masks": [slice(0.394, 0.396)], + }, + } + + model_settings = ModelSettings(model_init_kwargs=model_init_kwargs) + + model_context = ModelContext(diagnostics=diagnostics, + plasma=plasma_context.plasma, + equilibrium=data_context.equilibrium, + transforms=data_context.transforms, + model_settings = model_settings, + ) + + + + + + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=6, + sample_method="high_density", starting_samples=10, burn_frac=0.20, + stopping_criterion="mode", priors=bayes_settings.priors) + optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) + + + BayesWorkflowExample(tstart=tstart, tend=tend, dt=dt, + bayes_settings=bayes_settings, data_context=data_context, optimiser_context=optimiser_context, + plasma_context=plasma_context, model_context=ModelContext) From 14275613e24074231b9e7ba921e21632fc5f8965 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 12 Oct 2023 16:33:15 +0100 Subject: [PATCH 202/288] moved ln_prior to stand alone func --- indica/bayesblackbox.py | 82 ++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/indica/bayesblackbox.py b/indica/bayesblackbox.py index d2c15663..7b74f6e1 100644 --- a/indica/bayesblackbox.py +++ b/indica/bayesblackbox.py @@ -3,11 +3,9 @@ import numpy as np from scipy.stats import uniform - np.seterr(all="ignore") warnings.simplefilter("ignore", category=FutureWarning) - def gaussian(x, mean, sigma): return 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-1 / 2 * ((x - mean) / sigma) ** 2) @@ -17,6 +15,35 @@ def get_uniform(lower, upper): return uniform(loc=lower, scale=upper - lower) +def ln_prior(priors: dict, parameters: dict): + ln_prior = 0 + for prior_name, prior_func in priors.items(): + param_names_in_prior = [x for x in parameters.keys() if x in prior_name] + if param_names_in_prior.__len__() == 0: + # if prior assigned but no parameter then skip + continue + param_values = [parameters[x] for x in param_names_in_prior] + if hasattr(prior_func, "pdf"): + # for scipy.stats objects use pdf / for lambda functions just call + ln_prior += np.log(prior_func.pdf(*param_values)) + else: + # if lambda prior with 2+ args is defined when only 1 of + # its parameters is given ignore it + if prior_func.__code__.co_argcount != param_values.__len__(): + continue + else: + # Sorting to make sure args are given in the same order + # as the prior_name string + name_index = [ + prior_name.find(param_name_in_prior) + for param_name_in_prior in param_names_in_prior + ] + sorted_name_index, sorted_param_values = ( + list(x) for x in zip(*sorted(zip(name_index, param_values))) + ) + ln_prior += np.log(prior_func(*sorted_param_values)) + return ln_prior + class BayesBlackBox: """ Bayesian black box model that creates _ln_posterior function from plasma and diagnostic model objects @@ -42,8 +69,8 @@ def __init__( quant_to_optimise: list, priors: dict, - plasma_context=None, - model_handler= None, + plasma_context = None, + model_context = None, percent_error: float = 0.10, ): @@ -52,7 +79,7 @@ def __init__( self.priors = priors self.plasma_context = plasma_context - self.model_handler = model_handler + self.model_context = model_context self.percent_error = percent_error @@ -60,10 +87,10 @@ def __init__( if missing_data: # gives list of keys in quant_to_optimise but not data raise ValueError(f"{missing_data} not found in data given") - def _ln_likelihood(self): + def ln_likelihood(self): ln_likelihood = 0 for key in self.quant_to_optimise: - time_coord = self.plasma_context.time_to_calculate + time_coord = self.plasma_context.plasma.time_to_calculate model_data = self.bckc[key] exp_data = self.data[key].sel(t=time_coord) exp_error = exp_data * self.percent_error @@ -80,35 +107,6 @@ def _ln_likelihood(self): ln_likelihood += _ln_likelihood.mean(skipna=True).values return ln_likelihood - def _ln_prior(self, parameters: dict): - ln_prior = 0 - for prior_name, prior_func in self.priors.items(): - param_names_in_prior = [x for x in parameters.keys() if x in prior_name] - if param_names_in_prior.__len__() == 0: - # if prior assigned but no parameter then skip - continue - param_values = [parameters[x] for x in param_names_in_prior] - if hasattr(prior_func, "pdf"): - # for scipy.stats objects use pdf / for lambda functions just call - ln_prior += np.log(prior_func.pdf(*param_values)) - else: - # if lambda prior with 2+ args is defined when only 1 of - # its parameters is given ignore it - if prior_func.__code__.co_argcount != param_values.__len__(): - continue - else: - # Sorting to make sure args are given in the same order - # as the prior_name string - name_index = [ - prior_name.find(param_name_in_prior) - for param_name_in_prior in param_names_in_prior - ] - sorted_name_index, sorted_param_values = ( - list(x) for x in zip(*sorted(zip(name_index, param_values))) - ) - ln_prior += np.log(prior_func(*sorted_param_values)) - return ln_prior - def ln_posterior(self, parameters: dict, **kwargs): """ @@ -129,17 +127,17 @@ def ln_posterior(self, parameters: dict, **kwargs): model outputs from bckc and kinetic profiles """ - ln_prior = self._ln_prior(parameters) - if ln_prior == -np.inf: # Don't call models if outside priors + _ln_prior = ln_prior(self.priors, parameters) + if _ln_prior == -np.inf: # Don't call models if outside priors return -np.inf, {} self.plasma_context.update_profiles(parameters) - plasma_attributes = self.plasma_context.return_plasma_attributes() + plasma_attributes = self.plasma_context.return_plasma_attrs() - self.bckc = self.model_handler._build_bckc(parameters, **kwargs) # model calls + self.bckc = self.model_context._build_bckc(parameters, **kwargs) # model calls - ln_likelihood = self._ln_likelihood() # compare results to data - ln_posterior = ln_likelihood + ln_prior + _ln_likelihood = self.ln_likelihood() # compare results to data + ln_posterior = _ln_likelihood + _ln_prior blob = deepcopy({**self.bckc, **plasma_attributes}) return ln_posterior, blob From 33ec07a3b2478d63823b1f5f38b151d02f669af0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 12 Oct 2023 17:19:35 +0100 Subject: [PATCH 203/288] Initialisation fixed --- indica/workflows/bayes_workflow_example.py | 79 +++++++++++----------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index c14608a4..ffc131e3 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -274,8 +274,20 @@ def __post_init__(self): raise ValueError(f"{missing_quantities} missing the relevant diagnostic") +@dataclass +class PlasmaSettings: + tstart = None + tend = None + dt = None + main_ion = "h" + impurities = ("ar", "c") + impurity_concentration = (0.001, 0.04) + n_rad = 20 + + @dataclass class PlasmaContext: + plasma_settings: PlasmaSettings equilibrium: Equilibrium = None profile_params: dict = field(default_factory=lambda: DEFAULT_PROFILE_PARAMS) @@ -289,21 +301,17 @@ def init_plasma( tstart=None, tend=None, dt=None, - main_ion="h", - impurities=("ar", "c"), - impurity_concentration=(0.001, 0.04), - n_rad=20, ): self.plasma = Plasma( tstart=tstart, tend=tend, dt=dt, - main_ion=main_ion, - impurities=impurities, - impurity_concentration=impurity_concentration, + main_ion=self.plasma_settings.main_ion, + impurities=self.plasma_settings.impurities, + impurity_concentration=self.plasma_settings.impurity_concentration, full_run=False, - n_rad=n_rad, + n_rad=self.plasma_settings.n_rad, ) self.plasma.set_equilibrium(equilibrium) @@ -373,8 +381,6 @@ def save_phantom_profiles(self, kinetic_profiles=None, phantoms=None): } self.phantom_profiles = phantom_profiles - # - # # # def _init_fast_particles(self, run="RUN602", ): # @@ -429,7 +435,7 @@ def save_phantom_profiles(self, kinetic_profiles=None, phantoms=None): @dataclass class ModelContext: diagnostics: list - plasma: Plasma + plasma_context: PlasmaContext equilibrium: Equilibrium transforms: dict model_kwargs: Dict[str, Dict[str, Any]] @@ -440,7 +446,7 @@ class ModelContext: TODO: remove repeating code / likely make general methods """ - def update_model_kwargs(self, data): + def update_model_kwargs(self, data: dict): if self.model_kwargs is None: self.model_kwargs = {} for diag in diagnostics: @@ -459,8 +465,10 @@ def update_model_kwargs(self, data): self.model_kwargs["xrcs"]["background"] = background.mean(dim="wavelength") def init_models(self, ): - if not hasattr(self, "plasma"): - raise ValueError("needs plasma to setup_models") + if not hasattr(self, "plasma_context"): + raise ValueError("needs plasma_context to setup_models") + if not hasattr(self.plasma_context, "plasma"): + raise ValueError("plasma_context needs plasma to setup_models") self.models: Dict[str, Any] = {} for diag in self.diagnostics: @@ -495,7 +503,7 @@ def init_models(self, ): raise ValueError(f"{diag} not implemented in ModelHandler.setup_models") for model_name, model in self.models.items(): - model.plasma = self.plasma + model.plasma = self.plasma_context.plasma return self.models @@ -889,12 +897,10 @@ def __init__( self.tend = tend self.dt = dt - self.plasma_context.init_plasma(equilibrium=data_context.equilibrium, tstart=tstart, tend=tend, dt=dt, ) + self.plasma_context.init_plasma(equilibrium=self.data_context.equilibrium, tstart=self.tstart, tend=self.tend, dt=self.dt, ) self.plasma_context.save_phantom_profiles(phantoms=False) - - - self.model_context.update_model_kwargs(data_context.data) + self.model_context.update_model_kwargs(self.data_context.binned_data) self.model_context.init_models() self.model_call_kwargs = {"xrcs.pixel_offset": 4.0} @@ -1000,44 +1006,35 @@ def __call__( bayes_settings = BayesSettings(diagnostics=diagnostics, param_names=opt_params, opt_quantity=opt_quant, priors=DEFAULT_PRIORS, ) - data_settings = ReaderSettings(filters={}, - revisions={}) # Add general methods for filtering data co-ords to ReadST40 + data_settings = ReaderSettings(filters={}, revisions={}) # Add general methods for filtering data co-ords to ReadST40 data_context = ExpData(pulse=pulse, diagnostics=diagnostics, tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context.read_data() plasma_settings = PlasmaSettings() - plasma_context = PlasmaContext(plasma_settings = plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) + plasma_context = PlasmaContext(plasma_settings=plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) - - - model_init_kwargs = {"cxff_pi": {"element": "ar"}, - "cxff_tws_c": {"element": "c"}, - "xrcs": { - "window_masks": [slice(0.394, 0.396)], + model_init_kwargs = { + "cxff_pi": {"element": "ar"}, + "cxff_tws_c": {"element": "c"}, + "xrcs": { + "window_masks": [slice(0.394, 0.396)], }, } - model_settings = ModelSettings(model_init_kwargs=model_init_kwargs) - model_context = ModelContext(diagnostics=diagnostics, - plasma=plasma_context.plasma, - equilibrium=data_context.equilibrium, - transforms=data_context.transforms, - model_settings = model_settings, - ) - - - - + plasma_context=plasma_context, + equilibrium=data_context.equilibrium, + transforms=data_context.transforms, + model_kwargs = model_init_kwargs, + ) optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=6, sample_method="high_density", starting_samples=10, burn_frac=0.20, stopping_criterion="mode", priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) - BayesWorkflowExample(tstart=tstart, tend=tend, dt=dt, bayes_settings=bayes_settings, data_context=data_context, optimiser_context=optimiser_context, - plasma_context=plasma_context, model_context=ModelContext) + plasma_context=plasma_context, model_context=model_context) From 9b400442934241a509c494406a454f7a5e33f76b Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 09:38:57 +0100 Subject: [PATCH 204/288] fixed formatting/plotting of optimiser results and adjusted moments stopping criteria to normalise for chain length --- indica/workflows/bayes_workflow_example.py | 206 ++++++++++++--------- 1 file changed, 119 insertions(+), 87 deletions(-) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow_example.py index ffc131e3..8fbdef68 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow_example.py @@ -85,12 +85,12 @@ ), # impurity always more peaked "Te_prof.y0": get_uniform(1000, 5000), - "Te_prof.wped": get_uniform(2, 6), + "Te_prof.wped": get_uniform(1, 6), "Te_prof.wcenter": get_uniform(0.2, 0.4), "Te_prof.peaking": get_uniform(1, 4), # "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode "Ti_prof.y0": get_uniform(1000, 10000), - "Ti_prof.wped": get_uniform(2, 6), + "Ti_prof.wped": get_uniform(1, 6), "Ti_prof.wcenter": get_uniform(0.2, 0.4), "Ti_prof.peaking": get_uniform(1, 6), "xrcs.pixel_offset": get_uniform(-4.01, -4.0), @@ -172,10 +172,12 @@ ] -def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample=10, fractional_difference=0.001): +def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample=10, stopping_factor=10, debug=True): + # TODO: Compare old_chain to new_chain: if moments are different then keep going / convergence diagnostics here + autocorr = np.ones(shape=(iterations, n_params)) * np.nan old_mean = np.inf - old_std = np.inf + success_flag = False # requires succeeding check twice in a row for sample in sampler.sample( start_points, iterations=iterations, @@ -190,20 +192,29 @@ def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample dist_stats = describe(sampler.get_chain(flat=True)) new_mean = dist_stats.mean - new_std = np.sqrt(dist_stats.variance) - if all(np.abs(new_mean - old_mean) / old_mean < fractional_difference) and all( - np.abs(new_std - old_std) / old_std < fractional_difference): - break + dmean = np.abs(new_mean - old_mean) + nsamples = sampler.flatchain.shape[0] + + dmean_normed = dmean / old_mean * nsamples + + if debug: + print("") + print(f"dmean_normed: {dmean_normed.max()}") + if dmean_normed.max() < stopping_factor: + if success_flag: + break + else: + success_flag = True + else: + success_flag = False old_mean = new_mean - old_std = new_std autocorr = autocorr[ : sampler.iteration, ] return autocorr - def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sample=5, ): autocorr = np.ones(shape=(iterations, n_params)) * np.nan old_tau = np.inf @@ -243,14 +254,14 @@ def gelman_rubin(chain): return R -def dict_of_dataarray_to_numpy(self, dict_of_dataarray): +def dict_of_dataarray_to_numpy(dict_of_dataarray): """ Mutates input dictionary to change xr.DataArray objects to np.array """ for key, value in dict_of_dataarray.items(): if isinstance(value, dict): - self.dict_of_dataarray_to_numpy(value) + dict_of_dataarray_to_numpy(value) elif isinstance(value, xr.DataArray): dict_of_dataarray[key] = dict_of_dataarray[key].values return dict_of_dataarray @@ -276,19 +287,15 @@ def __post_init__(self): @dataclass class PlasmaSettings: - tstart = None - tend = None - dt = None - main_ion = "h" - impurities = ("ar", "c") - impurity_concentration = (0.001, 0.04) - n_rad = 20 + main_ion: str = "h" + impurities: Tuple[str] = ("ar", "c") + impurity_concentration: Tuple[float] = (0.001, 0.04) + n_rad: int = 20 @dataclass class PlasmaContext: plasma_settings: PlasmaSettings - equilibrium: Equilibrium = None profile_params: dict = field(default_factory=lambda: DEFAULT_PROFILE_PARAMS) """ @@ -560,11 +567,13 @@ class ReaderSettings: @dataclass class DataContext(ABC): reader_settings: ReaderSettings + pulse: int tstart: float tend: float dt: float transforms = None equilibrium = None + phantoms = False @abstractmethod def read_data(self, ): @@ -589,6 +598,7 @@ def process_data(self, model_context: ModelContext): class ExpData(DataContext): pulse: int diagnostics: list + phantoms = False """ Considering: either rewriting this class to take over from ReadST40 or vice versa @@ -638,8 +648,10 @@ def _process_data(self): @dataclass class MockData(DataContext): + pulse = None diagnostic_transforms: dict = field(default_factory=lambda: {}) model_call_params: dict = field(default_factory=lambda: {}) + phantoms = True def read_data(self): # Used with phantom data for purposes of tests @@ -653,7 +665,6 @@ def read_data(self): self.binned_data: dict = {} self.raw_data: dict = {} - def process_data(self, model_callables: Callable): self.model_callables = model_callables self.check_if_data_present(self._gen_data_strategy) @@ -673,6 +684,7 @@ class PhantomData(DataContext): pulse: int diagnostics: list model_call_kwargs: dict = field(default_factory=lambda: {}) + phantoms = True def read_data(self, ): self.reader = ReadST40(self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt, ) @@ -709,7 +721,7 @@ class OptimiserEmceeSettings: sample_method: str = "random" starting_samples: int = 100 stopping_criterion: str = "mode" - stopping_criterion_fract: float = 0.05 + stopping_criterion_factor: float = 10 @dataclass @@ -840,7 +852,7 @@ def run(self, ): self.optimiser_settings.iterations, self.optimiser_settings.param_names.__len__(), auto_sample=10, - fractional_difference=self.optimiser_settings.stopping_criterion_fract + stopping_factor=self.optimiser_settings.stopping_criterion_factor ) else: raise ValueError(f"Stopping criterion: {self.optimiser_settings.stopping_criterion} not recognised") @@ -868,8 +880,15 @@ def format_results(self): results["prior_sample"] = self.sample_from_priors( self.optimiser_settings.param_names, self.optimiser_settings.priors, size=int(1e4) ) - results["post_sample"] = self.optimiser.get_chain( + + post_sample = self.optimiser.get_chain( discard=int(self.optimiser.iteration * self.optimiser_settings.burn_frac), flat=True) + # pad index dim with maximum number of iterations + max_iter = self.optimiser_settings.iterations * self.optimiser_settings.nwalkers + npad = ((0, int(max_iter*(1-self.optimiser_settings.burn_frac) - post_sample.shape[0])), (0, 0)) + results["post_sample"] = np.pad(post_sample, npad, constant_values=np.nan) + + results["auto_corr"] = self.autocorr return results @@ -897,8 +916,9 @@ def __init__( self.tend = tend self.dt = dt - self.plasma_context.init_plasma(equilibrium=self.data_context.equilibrium, tstart=self.tstart, tend=self.tend, dt=self.dt, ) - self.plasma_context.save_phantom_profiles(phantoms=False) + self.plasma_context.init_plasma(equilibrium=self.data_context.equilibrium, tstart=self.tstart, tend=self.tend, + dt=self.dt, ) + self.plasma_context.save_phantom_profiles(phantoms=self.data_context.phantoms) self.model_context.update_model_kwargs(self.data_context.binned_data) self.model_context.init_models() @@ -912,12 +932,12 @@ def __init__( quant_to_optimise=self.bayes_settings.opt_quantity, priors=self.bayes_settings.priors) - self.optimiser_context.init_optimiser(self.blackbox.ln_posterior, model_kwargs={}) + self.optimiser_context.init_optimiser(self.blackbox.ln_posterior, model_kwargs=self.model_call_kwargs) def __call__( self, filepath="./results/test/", - run=None, + run="RUN01", mds_write=False, pulse_to_write=None, plot=False, @@ -925,47 +945,40 @@ def __call__( ): self.result = self._build_inputs_dict() - results = [] for time in self.plasma_context.time_iterator(): self.plasma_context.plasma.time_to_calculate = time - print(f"Time: {self.plasma_context.plasma.time_to_calculate}") + print(f"Time: {time.values:.2f}") self.optimiser_context.sample_start_points() self.optimiser_context.run() results.append(self.optimiser_context.format_results()) self.optimiser_context.optimiser.reset() - # result = self._build_result_dict() - - # Combine the blobs/results from optimiser and add time dim - - profiles = {} - globals = {} - for key, prof in results[0]["PROFILES"].items(): - if key == "RHO_POLOIDAL": - profiles[key] = results[0]["PROFILES"]["RHO_POLOIDAL"] - elif key == "RHO_TOR": - profiles[key] = results[0]["PROFILES"]["RHO_TOR"] - else: - _profs = [result["PROFILES"][key] for result in results] - profiles[key] = xr.concat(_profs, self.tsample) - - for key, prof in results[0]["GLOBAL"].items(): - _glob = [result["GLOBAL"][key] for result in results] - globals[key] = xr.concat(_glob, self.tsample) - - result = {"PROFILES": profiles, "GLOBAL": globals} + # unpack results and add time axis + blobs = {} + for key in results[0]["blobs"].keys(): + _blob = [result["blobs"][key] for result in results] + blobs[key] = xr.concat(_blob, self.plasma_context.plasma.t) + self.blobs = blobs - self.result = dict(self.result, **result, ) - _result = dict(result, **self.result) + opt_samples = {} + for key in results[0].keys(): + if key == "blobs": + continue + _opt_samples = [result[key] for result in results] + opt_samples[key] = np.array(_opt_samples) + self.opt_samples = opt_samples - self.result = self.dict_of_dataarray_to_numpy(self.result) + result = self._build_result_dict() + self.result = dict(self.result, **result) - self.save_pickle(_result, filepath=filepath, ) + self.save_pickle(self.result, filepath=filepath, ) if plot: # currently requires result with DataArrays - plot_bayes_result(_result, filepath) + plot_bayes_result(self.result, filepath) + + self.result = dict_of_dataarray_to_numpy(self.result) if mds_write: # check_analysis_run(self.pulse, self.run) @@ -973,7 +986,7 @@ def __call__( pulse_to_write=pulse_to_write, run=run, diagnostic_quantities=self.bayes_settings.opt_quantity, - mode="EDIT", + mode="NEW", ) write_nodes(pulse_to_write, self.node_structure, self.result) @@ -981,60 +994,79 @@ def __call__( if __name__ == "__main__": - # mock_transforms = {"xrcs": helike_transform_example(1), - # "smmh1": smmh1_transform_example(1), - # "cxff_pi": pi_transform_example(5), - # "ts": ts_transform_example(11), } - - pulse = 11216 - tstart = 0.05 - tend = 0.06 + pulse = 11336 + tstart = 0.07 + tend = 0.08 dt = 0.01 diagnostics = [ - "xrcs", - "efit", + # "xrcs", + # "efit", # "smmh1", # "cxff_pi", - # "ts", + "ts", ] - opt_quant = ["xrcs.spectra", "efit.wp"] - opt_params = ["Te_prof.y0", - "Ti_prof.y0", - "Ne_prof.y0"] - + # diagnostic_quantities + opt_quant = [ + # "xrcs.spectra", + # "efit.wp", + "ts.te" + ] + opt_params = [ + "Te_prof.y0", + "Te_prof.peaking", + "Te_prof.wped", + "Te_prof.wcenter", + + # "Ti_prof.y0", + # "Ne_prof.y0", + ] + + # BlackBoxSettings bayes_settings = BayesSettings(diagnostics=diagnostics, param_names=opt_params, opt_quantity=opt_quant, priors=DEFAULT_PRIORS, ) - data_settings = ReaderSettings(filters={}, revisions={}) # Add general methods for filtering data co-ords to ReadST40 + data_settings = ReaderSettings(filters={}, + revisions={}) # Add general methods for filtering data co-ords to ReadST40 + + # mock_transforms = {"xrcs": helike_transform_example(1), + # "smmh1": smmh1_transform_example(1), + # "cxff_pi": pi_transform_example(5), + # "ts": ts_transform_example(11), } + # data_context = MockData(pulse=None, diagnostic_transforms=mock_transforms, ) data_context = ExpData(pulse=pulse, diagnostics=diagnostics, tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context.read_data() - plasma_settings = PlasmaSettings() + plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), + n_rad=20) plasma_context = PlasmaContext(plasma_settings=plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) model_init_kwargs = { - "cxff_pi": {"element": "ar"}, - "cxff_tws_c": {"element": "c"}, - "xrcs": { - "window_masks": [slice(0.394, 0.396)], - }, - } + "cxff_pi": {"element": "ar"}, + "cxff_tws_c": {"element": "c"}, + "xrcs": { + "window_masks": [slice(0.394, 0.396)], + }, + } model_context = ModelContext(diagnostics=diagnostics, plasma_context=plasma_context, equilibrium=data_context.equilibrium, transforms=data_context.transforms, - model_kwargs = model_init_kwargs, + model_kwargs=model_init_kwargs, ) - optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=6, - sample_method="high_density", starting_samples=10, burn_frac=0.20, - stopping_criterion="mode", priors=bayes_settings.priors) + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=500, + sample_method="high_density", starting_samples=100, burn_frac=0.20, + stopping_criterion="mode", stopping_criterion_factor=10, + priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) - BayesWorkflowExample(tstart=tstart, tend=tend, dt=dt, - bayes_settings=bayes_settings, data_context=data_context, optimiser_context=optimiser_context, - plasma_context=plasma_context, model_context=model_context) + workflow = BayesWorkflowExample(tstart=tstart, tend=tend, dt=dt, + bayes_settings=bayes_settings, data_context=data_context, + optimiser_context=optimiser_context, + plasma_context=plasma_context, model_context=model_context) + + workflow(pulse_to_write=25000000, run="RUN01", mds_write=True, plot=True, filepath="./results/test_moments/") From 1bc37b1f0a0ca43e1ad405bd10c2b9f10b16a88c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 09:39:26 +0100 Subject: [PATCH 205/288] TS dimension bug fix added --- indica/readers/st40reader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/indica/readers/st40reader.py b/indica/readers/st40reader.py index af54c9ee..7855415e 100644 --- a/indica/readers/st40reader.py +++ b/indica/readers/st40reader.py @@ -3,6 +3,7 @@ """ +from copy import deepcopy from typing import Any from typing import Dict from typing import List @@ -1092,6 +1093,9 @@ def _get_thomson_scattering( y, y_path = self._get_signal(uid, instrument, ":y", revision) z, z_path = self._get_signal(uid, instrument, ":z", revision) R, R_path = self._get_signal(uid, instrument, ":R", revision) + z = R * 0.0 + x = deepcopy(R) + y = 0 for q in quantities: qval, q_path = self._get_signal( From bc5634e2992de6bddb802044f6e805095598cc96 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 09:40:02 +0100 Subject: [PATCH 206/288] formatting for results dict fixed --- indica/workflows/abstract_bayes_workflow.py | 136 ++++++++++---------- 1 file changed, 66 insertions(+), 70 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 27ec77dd..8c929ac0 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -41,10 +41,9 @@ def _build_inputs_dict(self): """ result = {} - quant_list = [item.split(".") for item in self.opt_quantity] + quant_list = [item.split(".") for item in self.bayes_settings.opt_quantity] - result["TIME_BINS"] = self.plasma.t - result["TIME"] = self.plasma.time_to_calculate + result["TIME"] = self.plasma_context.plasma.t result["METADATA"] = { "GITCOMMIT": "PLACEHOLDER", @@ -53,48 +52,46 @@ def _build_inputs_dict(self): } result["INPUT"] = { - "BURN_FRAC": self.burn_frac, - "ITER": self.iterations, - "NWALKERS": self.nwalkers, - "MODEL_KWARGS": self.model_kwargs, - "OPT_QUANTITY": self.opt_quantity, - "PARAM_NAMES": self.param_names, - "PULSE": self.pulse, - "IMPURITIES": self.plasma.impurities, - "MAIN_ION": self.plasma.main_ion, + "BURN_FRAC": self.optimiser_context.optimiser_settings.burn_frac, + "ITER": self.optimiser_context.optimiser_settings.iterations, + "NWALKERS": self.optimiser_context.optimiser_settings.nwalkers, + "MODEL_KWARGS": self.model_call_kwargs, + "OPT_QUANTITY": self.bayes_settings.opt_quantity, + "PARAM_NAMES": self.bayes_settings.param_names, + "PULSE": self.data_context.pulse, + "IMPURITIES": self.plasma_context.plasma_settings.impurities, + "MAIN_ION": self.plasma_context.plasma_settings.main_ion, "TSTART":self.tstart, "TEND": self.tend, "DT": self.dt, - "TSAMPLE": self.tsample, } result["INPUT"]["WORKFLOW"] = { diag_name.upper(): { - "PULSE": self.pulse, # Change this if different pulses used + "PULSE": self.data_context.pulse, # Change this if different pulses used "USAGE": "".join( [quantity[1] for quantity in quant_list if quantity[0] == diag_name] ), "RUN": "PLACEHOLDER", } - for diag_name in self.diagnostics + for diag_name in self.bayes_settings.diagnostics } result["DIAG_DATA"] = { diag_name.upper(): { - quantity[1].upper(): self.opt_data[f"{quantity[0]}.{quantity[1]}"] + quantity[1].upper(): self.data_context.opt_data[f"{quantity[0]}.{quantity[1]}"] for quantity in quant_list if quantity[0] == diag_name } - for diag_name in self.diagnostics + for diag_name in self.bayes_settings.diagnostics } - return result def _build_result_dict(self, ): result = {} - quant_list = [item.split(".") for item in self.opt_quantity] + quant_list = [item.split(".") for item in self.bayes_settings.opt_quantity] result["MODEL_DATA"] = { diag_name.upper(): { @@ -102,61 +99,61 @@ def _build_result_dict(self, ): for quantity in quant_list if quantity[0] == diag_name } - for diag_name in self.diagnostics + for diag_name in self.bayes_settings.diagnostics } - result["MODEL_DATA"]["SAMPLES"] = self.samples - + result["MODEL_DATA"]["SAMPLE_IDX"] = np.arange(self.optimiser_context.optimiser_settings.iterations* + (1-self.optimiser_context.optimiser_settings.burn_frac)) result["PHANTOMS"] = { - "FLAG": self.phantoms, - "NE": self.phantom_profiles["electron_density"], - "TE": self.phantom_profiles["electron_temperature"], - "TI": self.phantom_profiles["ion_temperature"].sel( - element=self.plasma.main_ion + "FLAG": self.data_context.phantoms, + "NE": self.plasma_context.phantom_profiles["electron_density"], + "TE": self.plasma_context.phantom_profiles["electron_temperature"], + "TI": self.plasma_context.phantom_profiles["ion_temperature"].sel( + element=self.plasma_context.plasma_settings.main_ion ), - "NI": self.phantom_profiles["ion_density"].sel( - element=self.plasma.main_ion + "NI": self.plasma_context.phantom_profiles["ion_density"].sel( + element=self.plasma_context.plasma_settings.main_ion ), - "NNEUTR": self.phantom_profiles["neutral_density"], - "NFAST": self.phantom_profiles["fast_density"], + "NNEUTR": self.plasma_context.phantom_profiles["neutral_density"], + "NFAST": self.plasma_context.phantom_profiles["fast_density"], } result["PHANTOMS"].update( { - f"NIZ{num_imp + 1}": self.phantom_profiles["impurity_density"].sel( + f"NIZ{num_imp + 1}": self.plasma_context.phantom_profiles["impurity_density"].sel( element=imp ) - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) } ) result["PHANTOMS"].update( { - f"TIZ{num_imp + 1}": self.phantom_profiles["ion_temperature"].sel( + f"TIZ{num_imp + 1}": self.plasma_context.phantom_profiles["ion_temperature"].sel( element=imp ) - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) } ) result["PROFILES"] = { - "RHO_POLOIDAL": self.plasma.rho, - "RHO_TOR": self.plasma.equilibrium.rhotor.interp(t=self.plasma.t), + "RHO_POLOIDAL": self.plasma_context.plasma.rho, + "RHO_TOR": self.plasma_context.plasma.equilibrium.rhotor.interp(t=self.plasma_context.plasma.t), "NE": self.blobs["electron_density"].median(dim="index"), "NI": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) + .sel(element=self.plasma_context.plasma_settings.main_ion) .median(dim="index"), "TE": self.blobs["electron_temperature"].median(dim="index"), "TI": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) + .sel(element=self.plasma_context.plasma_settings.main_ion) .median(dim="index"), "NFAST": self.blobs["fast_density"].median(dim="index"), "NNEUTR": self.blobs["neutral_density"].median(dim="index"), "NE_ERR": self.blobs["electron_density"].std(dim="index"), "NI_ERR": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) + .sel(element=self.plasma_context.plasma_settings.main_ion) .std(dim="index"), "TE_ERR": self.blobs["electron_temperature"].std(dim="index"), "TI_ERR": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) + .sel(element=self.plasma_context.plasma_settings.main_ion) .std(dim="index"), "NFAST_ERR": self.blobs["fast_density"].std(dim="index"), "NNEUTR_ERR": self.blobs["neutral_density"].std(dim="index"), @@ -169,7 +166,7 @@ def _build_result_dict(self, ): f"NIZ{num_imp + 1}": self.blobs["impurity_density"] .sel(element=imp) .median(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } result["PROFILES"] = { @@ -178,7 +175,7 @@ def _build_result_dict(self, ): f"NIZ{num_imp + 1}_ERR": self.blobs["impurity_density"] .sel(element=imp) .std(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } result["PROFILES"] = { @@ -187,7 +184,7 @@ def _build_result_dict(self, ): f"TIZ{num_imp + 1}": self.blobs["ion_temperature"] .sel(element=imp) .median(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } result["PROFILES"] = { @@ -196,17 +193,18 @@ def _build_result_dict(self, ): f"TIZ{num_imp + 1}_ERR": self.blobs["ion_temperature"] .sel(element=imp) .std(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } result["PROFILE_STAT"] = { - "SAMPLES": self.samples, - "RHO_POLOIDAL": self.plasma.rho, + "SAMPLE_IDX": np.arange(self.optimiser_context.optimiser_settings.iterations * + (1-self.optimiser_context.optimiser_settings.burn_frac)), + "RHO_POLOIDAL": self.plasma_context.plasma.rho, "NE": self.blobs["electron_density"], - "NI": self.blobs["ion_density"].sel(element=self.plasma.main_ion), + "NI": self.blobs["ion_density"].sel(element=self.plasma_context.plasma_settings.main_ion), "TE": self.blobs["electron_temperature"], - "TI": self.blobs["ion_temperature"].sel(element=self.plasma.main_ion), + "TI": self.blobs["ion_temperature"].sel(element=self.plasma_context.plasma_settings.main_ion), "NFAST": self.blobs["fast_density"], "NNEUTR": self.blobs["neutral_density"], } @@ -214,28 +212,28 @@ def _build_result_dict(self, ): **result["PROFILE_STAT"], **{ f"NIZ{num_imp + 1}": self.blobs["impurity_density"].sel(element=imp) - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } result["PROFILE_STAT"] = { **result["PROFILE_STAT"], **{ f"TIZ{num_imp + 1}": self.blobs["ion_temperature"].sel(element=imp) - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } result["OPTIMISATION"] = { - "ACCEPT_FRAC": self.accept_frac, - "PRIOR_SAMPLE": self.prior_sample, - "POST_SAMPLE": self.post_sample, - "AUTOCORR": self.autocorr, - # "GELMANRUBIN": gelman_rubin(self.sampler.get_chain(flat=False)) + "ACCEPT_FRAC": self.opt_samples["accept_frac"], + "PRIOR_SAMPLE": self.opt_samples["prior_sample"], + "POST_SAMPLE": self.opt_samples["post_sample"], + "AUTO_CORR": self.opt_samples["auto_corr"], + # "GELMAN_RUBIN": gelman_rubin(self.sampler.get_chain(flat=False)) } result["GLOBAL"] = { "TI0": self.blobs["ion_temperature"] - .sel(element=self.plasma.main_ion) + .sel(element=self.plasma_context.plasma_settings.main_ion) .sel(rho_poloidal=0, method="nearest") .median(dim="index"), "TE0": self.blobs["electron_temperature"] @@ -245,7 +243,7 @@ def _build_result_dict(self, ): .sel(rho_poloidal=0, method="nearest") .median(dim="index"), "NI0": self.blobs["ion_density"] - .sel(element=self.plasma.main_ion) + .sel(element=self.plasma_context.plasma_settings.main_ion) .sel(rho_poloidal=0, method="nearest") .median(dim="index"), "WP": self.blobs["wp"] @@ -272,7 +270,7 @@ def _build_result_dict(self, ): .sel(element=imp) .sel(rho_poloidal=0, method="nearest") .median(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } result["GLOBAL"] = { @@ -282,7 +280,7 @@ def _build_result_dict(self, ): .sel(element=imp) .sel(rho_poloidal=0, method="nearest") .std(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } result["GLOBAL"] = { @@ -292,7 +290,7 @@ def _build_result_dict(self, ): .sel(element=imp) .sel(rho_poloidal=0, method="nearest") .median(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } result["GLOBAL"] = { @@ -302,24 +300,22 @@ def _build_result_dict(self, ): .sel(element=imp) .sel(rho_poloidal=0, method="nearest") .std(dim="index") - for num_imp, imp in enumerate(self.plasma.impurities) + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } return result - def save_pickle(self, result, filepath): - if filepath: - Path(filepath).mkdir(parents=True, exist_ok=True) - with open(filepath + "results.pkl", "wb") as handle: - pickle.dump(result, handle) + Path(filepath).mkdir(parents=True, exist_ok=True) + with open(filepath + "results.pkl", "wb") as handle: + pickle.dump(result, handle) @abstractmethod def __call__(self, filepath="./results/test/", **kwargs): - self.run_sampler() - self.save_pickle(filepath=filepath) - self.result = self.dict_of_dataarray_to_numpy(self.result) + result = self.run_sampler() + self.save_pickle(result, filepath=filepath) + self.result = result return self.result From 365e4d8e302addac9960428477d675e9089716da Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 09:40:49 +0100 Subject: [PATCH 207/288] Now plots for all time points --- indica/workflows/bayes_plots.py | 483 ++++++++++++++++---------------- 1 file changed, 236 insertions(+), 247 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index 15d61da1..b70cb92d 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -5,16 +5,17 @@ import flatdict import matplotlib.pyplot as plt import numpy as np +import xarray as xr from indica.utilities import set_axis_sci from indica.utilities import set_plot_rcparams - +from pathlib import Path def plot_profile( - profile, - blobkey: str, + profile: xr.DataArray, + phantom_profile: xr.DataArray, + label: str, figheader="./results/test/", - phantom_profile=None, logscale=False, sharefig=False, filename="", @@ -28,17 +29,17 @@ def plot_profile( profile.rho_poloidal, profile.quantile(0.16, dim="index"), profile.quantile(0.84, dim="index"), - label=f"{blobkey}, 68% Confidence", + label=f"{label}, 68% Confidence", zorder=3, color=color, alpha=0.9, ) - if blobkey != "NFAST": + if label != "NFAST": plt.fill_between( profile.rho_poloidal, profile.quantile(0.025, dim="index"), profile.quantile(0.975, dim="index"), - label=f"{blobkey}, 95% Confidence", + label=f"{label}, 95% Confidence", zorder=2, color="grey", alpha=0.4, @@ -47,19 +48,15 @@ def plot_profile( profile.rho_poloidal, profile.quantile(0.005, dim="index"), profile.quantile(0.995, dim="index"), - label=f"{blobkey}, Max-Min", + label=f"{label}, Max-Min", zorder=1, color="lightgrey", alpha=0.2, ) - if phantom_profile["FLAG"]: - if "element" in phantom_profile[blobkey].dims: - phantom = phantom_profile[blobkey].sel(element="ar") - else: - phantom = phantom_profile[blobkey] - phantom.plot( - label=f"{blobkey}, phantom profile", + if phantom_profile.any(): + phantom_profile.plot( + label=f"{label}, phantom profile", linestyle=linestyle, color="black", zorder=4, @@ -78,16 +75,16 @@ def plot_profile( if filename: plt.savefig(figheader + f"{filename}{filetype}") else: - plt.savefig(figheader + f"{blobkey}{filetype}") + plt.savefig(figheader + f"{label}{filetype}") plt.close("all") def _plot_1d( - blobs: dict, - blobkey: str, - diag_data: dict, + data: xr.DataArray, + diag_data: xr.DataArray, filename: str, figheader="./results/test/", + label = "", ylabel="a.u.", xlabel="[]", xlim=None, @@ -95,50 +92,46 @@ def _plot_1d( hide_legend=False, **kwargs, ): - if blobkey not in blobs.keys(): - raise ValueError(f"{blobkey} not in blobs") - plt.figure(figsize=figsize) - blob_data = blobs[blobkey] - dims = tuple(name for name in blob_data.dims if name != "index") + dims = tuple(name for name in data.dims if name != "index") plt.fill_between( - blob_data.__getattr__(dims[0]), - blob_data.quantile(0.16, dim="index"), - blob_data.quantile(0.84, dim="index"), - label=f"{blobkey}, 68% Confidence", + data.__getattr__(dims[0]), + data.quantile(0.16, dim="index"), + data.quantile(0.84, dim="index"), + label=f"{label}, 68% Confidence", zorder=3, color="blue", ) plt.fill_between( - blob_data.__getattr__(dims[0]), - blob_data.quantile(0.025, dim="index"), - blob_data.quantile(0.975, dim="index"), - label=f"{blobkey}, 95% Confidence", + data.__getattr__(dims[0]), + data.quantile(0.025, dim="index"), + data.quantile(0.975, dim="index"), + label=f"{label}, 95% Confidence", zorder=2, color="grey", ) plt.fill_between( - blob_data.__getattr__(dims[0]), - blob_data.quantile(0.005, dim="index"), - blob_data.quantile(0.995, dim="index"), - label=f"{blobkey}, Max-Min", + data.__getattr__(dims[0]), + data.quantile(0.005, dim="index"), + data.quantile(0.995, dim="index"), + label=f"{label}, Max-Min", zorder=1, color="lightgrey", ) if "channel" in dims: plt.plot( - diag_data[blobkey].__getattr__(dims[0]), - diag_data[blobkey].sel(t=blob_data.t).values, + diag_data.__getattr__(dims[0]), + diag_data, "k^", - label=f"{blobkey} data", + label=f"{label} data", zorder=4, ) else: plt.plot( - diag_data[blobkey].__getattr__(dims[0]), - diag_data[blobkey].sel(t=blob_data.t).values, + diag_data.__getattr__(dims[0]), + diag_data, "k-", - label=f"{blobkey} data", + label=f"{label} data", zorder=4, ) @@ -154,10 +147,10 @@ def _plot_1d( def violinplot( - data: dict, - key: str, - diag_data: dict, + data, + diag_data, filename: str, + xlabel="", ylabel="[a.u.]", figheader="./results/test/", **kwargs, @@ -167,23 +160,25 @@ def violinplot( 1, 1, ) - _data = data[key].values - _data = _data[ - ((_data > np.quantile(_data, 0.16)) & (_data < np.quantile(_data, 0.84))) + _data = data[ + ((data > np.quantile(data, 0.16)) & (data < np.quantile(data, 0.84))) ] + if hasattr(diag_data, "error"): + diag_data_err = diag_data.error + else: + diag_data_err = diag_data * 0.10 + violin = axs.violinplot( _data, showextrema=False, # quantiles=[0.025, 0.975, 0.16, 0.84], # showmedians=True, ) - y = diag_data[key].sel(t=data[key].t).values - # TODO abstract the yerr axs.errorbar( - 1, y=y, yerr=y * 0.10, fmt="D", ecolor="black", capsize=10, color="black" + 1, y=diag_data, yerr=diag_data_err, fmt="D", ecolor="black", capsize=10, color="black" ) violin["bodies"][0].set_edgecolor("black") - axs.set_xlabel(key) + axs.set_xlabel(xlabel) top = axs.get_ylim()[1] # bot = axs.get_ylim()[0] axs.set_ylim(top=top * 1.1, bottom=0) @@ -242,12 +237,16 @@ def plot_autocorr(autocorr, param_names, figheader, filetype=".png"): def plot_bayes_result( results, - figheader="./results/test/", + filepath="./results/test/", filetype=".png", **kwargs, ): - # delete all but pickle in directory - for root, dirs, files in os.walk(figheader): + # delete all but pickle in directory and remove empty directories + for root, dirs, files in os.walk(filepath): + for dir in dirs: + if not os.listdir(root+dir): + print(f"Deleting {os.path.join(root, dir)}") + os.rmdir(os.path.join(root, dir)) for f in files: if f.endswith(".pkl"): continue @@ -255,237 +254,227 @@ def plot_bayes_result( print(f"Deleting {os.path.join(root, f)}") os.remove(os.path.join(root, f)) + # Create time directories + time = results["TIME"] + for t in time.values: + Path(filepath+f"/t:{t:.2f}").mkdir(parents=True, exist_ok=True) + + diag_data = flatdict.FlatDict(results["DIAG_DATA"], ".") model_data = flatdict.FlatDict(results["MODEL_DATA"], ".") profiles = flatdict.FlatDict(results["PROFILE_STAT"], ".") post_sample = results["OPTIMISATION"]["POST_SAMPLE"] prior_sample = results["OPTIMISATION"]["PRIOR_SAMPLE"] - autocorr = results["OPTIMISATION"]["AUTOCORR"] + auto_corr = results["OPTIMISATION"]["AUTO_CORR"] param_names = results["INPUT"]["PARAM_NAMES"] phantom_profiles = flatdict.FlatDict(results["PHANTOMS"], ".") - plot_autocorr(autocorr, param_names, figheader, filetype=filetype) + # select time index for plotting + for t_idx, t in enumerate(time): + figheader = filepath+f"t:{t.values:.2f}/" + + plot_autocorr(auto_corr[t_idx,], param_names, figheader, filetype=filetype) + # set_plot_rcparams("multi") - if "CXFF_PI.TI" in model_data.keys(): - max_channel = diag_data["CXFF_PI.TI"].sel(t=model_data["CXFF_PI.TI"].t).idxmax("channel").values - model_data["CXFF_PI.TI0"] = model_data["CXFF_PI.TI"].sel( - channel=max_channel + # check if model key exists + + key = "EFIT.WP" + if key in model_data.keys(): + violinplot( + model_data[key].sel(t=t), + diag_data[key].sel(t=t), + f"{key.replace('.', '_')}" + filetype, + xlabel=key, + figheader=figheader, + ylabel="Energy [J]", + ) + key = "SMMH1.NE" + if key in model_data.keys(): + violinplot( + model_data[key].sel(t=t), + diag_data[key].sel(t=t), + f"{key.replace('.', '_')}" + filetype, + figheader=figheader, + xlabel=key, + ylabel=r"Line Integrated Density [$m^{-2}$]", + ) + key = "XRCS.TE_KW" + if key in model_data.keys(): + violinplot( + model_data[key].sel(t=t), + diag_data[key].sel(t=t), + f"{key.replace('.', '_')}" + filetype, + figheader=figheader, + xlabel=key, + ylabel="Temperature [eV]", + ) + key = "XRCS.TI_W" + if key in model_data.keys(): + violinplot( + model_data[key].sel(t=t), + diag_data[key].sel(t=t), + f"{key.replace('.', '_')}" + filetype, + figheader=figheader, + xlabel = key, + ylabel="Temperature [eV]", ) - diag_data["CXFF_PI.TI0"] = diag_data["CXFF_PI.TI"].sel( - channel=max_channel - ) - model_data["CXFF_PI.TI0"] = model_data["CXFF_PI.TI"].sel( - channel=diag_data["CXFF_PI.TI"].channel - ) - key = "CXFF_PI.TI0" - if key in model_data.keys(): - violinplot( - model_data, - key, - diag_data, - f"{key.replace('.', '_')}" + filetype, - figheader=figheader, - ylabel="Temperature [eV]", - ) + key = "XRCS.SPECTRA" + if key in model_data.keys(): + _plot_1d( + model_data[key].sel(t=t), + diag_data[key].sel(t=t), + f"{key.replace('.', '_')}" + filetype, + label=key, + figheader=figheader, + ylabel="Intensity [a.u.]", + xlabel="Wavelength [nm]", + xlim=(0.394, 0.401), + figsize=(6, 4), + ) + key = "CXFF_PI.TI" + if key in model_data.keys(): + _plot_1d( + model_data[key].sel(t=t), + diag_data[key].sel(t=t), + f"{key.replace('.', '_')}" + filetype, + label=key, + figheader=figheader, + ylabel="Temperature [eV]", + xlabel="Channel", + hide_legend=True, + ) + key = "CXFF_TWS_C.TI" + if key in model_data.keys(): + _plot_1d( + model_data[key].sel(t=t), + diag_data[key].sel(t=t), + f"{key.replace('.', '_')}" + filetype, + label=key, + figheader=figheader, + ylabel="Temperature [eV]", + xlabel="Channel", + hide_legend=True, + + ) - key = "EFIT.WP" - if key in model_data.keys(): - violinplot( - model_data, + key = "TS.TE" + if key in model_data.keys(): + _plot_1d( + model_data[key].sel(t=t), + diag_data[key].sel(t=t), + f"{key.replace('.', '_')}" + filetype, + label=key, + figheader=figheader, + ylabel="Temperature [eV]", + xlabel="Channel", + ) + key = "TS.NE" + if key in model_data.keys(): + _plot_1d( + model_data[key].sel(t=t), + diag_data[key].sel(t=t), + f"{key.replace('.', '_')}" + filetype, + label=key, + figheader=figheader, + ylabel="Density [m^-3]", + xlabel="Channel", + ) + + key = "TE" + plot_profile( + profiles[key].sel(t=t), + phantom_profiles[key].sel(t=t), key, - diag_data, - f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="Energy [J]", + filetype=filetype, + sharefig=True, + color="blue", + linestyle="dashdot", ) - key = "SMMH1.NE" - if key in model_data.keys(): - violinplot( - model_data, + key = "TI" + plot_profile( + profiles[key].sel(t=t, ), + phantom_profiles[key].sel(t=t, ), key, - diag_data, - f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel=r"Line Integrated Density [$m^{-2}$]", + filename="temperature", + filetype=filetype, + color="red", + linestyle="dotted", ) - key = "XRCS.TE_KW" - if key in model_data.keys(): - violinplot( - model_data, - key, - diag_data, - f"{key.replace('.', '_')}" + filetype, - figheader=figheader, - ylabel="Temperature [eV]", - ) - key = "XRCS.TI_W" - if key in model_data.keys(): - violinplot( - model_data, + + key = "NE" + plot_profile( + profiles[key].sel(t=t), + phantom_profiles[key].sel(t=t), key, - diag_data, - f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="Temperature [eV]", + filetype=filetype, + color="blue", + sharefig=True, ) - - set_plot_rcparams("multi") - key = "XRCS.SPECTRA" - if key in model_data.keys(): - _plot_1d( - model_data, + key = "NI" + plot_profile( + profiles[key].sel(t=t), + phantom_profiles[key].sel(t=t), key, - diag_data, - f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="Intensity [a.u.]", - xlabel="Wavelength [nm]", - xlim=(0.394, 0.401), - figsize=(6, 4), + filetype=filetype, + sharefig=True, + color="red", ) - key = "CXFF_PI.TI" - if key in model_data.keys(): - _plot_1d( - model_data, + key = "NFAST" + plot_profile( + profiles[key].sel(t=t), + phantom_profiles[key].sel(t=t), key, - diag_data, - f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="Temperature [eV]", - xlabel="Channel", - hide_legend=True, + filename="densities", + filetype=filetype, + color="green", ) - key = "CXFF_TWS_C.TI" - if key in model_data.keys(): - _plot_1d( - model_data, + + key = "NIZ1" + plot_profile( + profiles[key].sel(t=t), + phantom_profiles[key].sel(t=t), key, - diag_data, - f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="Temperature [eV]", - xlabel="Channel", - hide_legend=True, - + filename=f"{profiles[key].element.values} density", + filetype=filetype, + color="red", ) - key = "TS.TE" - if key in model_data.keys(): - _plot_1d( - model_data, + key = "NIZ2" + plot_profile( + profiles[key].sel(t=t), + phantom_profiles[key].sel(t=t), key, - diag_data, - f"{key.replace('.', '_')}" + filetype, figheader=figheader, - ylabel="Temperature [eV]", - xlabel="Channel", + filename=f"{profiles[key].element.values} density", + filetype=filetype, + color="red", ) - key = "TS.NE" - if key in model_data.keys(): - _plot_1d( - model_data, + + key = "NNEUTR" + plot_profile( + profiles[key].sel(t=t), + phantom_profiles[key].sel(t=t), key, - diag_data, - f"{key.replace('.', '_')}" + filetype, + filename="neutral density", figheader=figheader, - ylabel="Density [m^-3]", - xlabel="Channel", + filetype=filetype, + logscale=True, ) - key = "TE" - plot_profile( - profiles[key], - key, - figheader=figheader, - filetype=filetype, - phantom_profile=phantom_profiles, - sharefig=True, - color="blue", - linestyle="dashdot", - ) - key = "TI" - plot_profile( - profiles[key], - key, - figheader=figheader, - filename="temperature", - filetype=filetype, - phantom_profile=phantom_profiles, - color="red", - linestyle="dotted", - ) - - key = "NE" - plot_profile( - profiles[key], - key, - figheader=figheader, - filetype=filetype, - phantom_profile=phantom_profiles, - color="blue", - sharefig=True, - ) - key = "NI" - plot_profile( - profiles[key], - key, - figheader=figheader, - filetype=filetype, - phantom_profile=phantom_profiles, - sharefig=True, - color="red", - ) - key = "NFAST" - plot_profile( - profiles[key], - key, - figheader=figheader, - filename="densities", - filetype=filetype, - phantom_profile=phantom_profiles, - color="green", - ) + post_sample_filtered = post_sample[t_idx,][~np.isnan(post_sample[t_idx,]).any(axis=1)] + corner.corner(post_sample_filtered, labels=param_names) + plt.savefig(figheader + "posterior" + filetype) - key = "NIZ1" - plot_profile( - profiles[key], - key, - figheader=figheader, - filename="ar density", - filetype=filetype, - phantom_profile=phantom_profiles, - color="red", - ) - - key = "NIZ2" - plot_profile( - profiles[key], - key, - figheader=figheader, - filename="c density", - filetype=filetype, - phantom_profile=phantom_profiles, - color="red", - ) - - key = "NNEUTR" - plot_profile( - profiles[key], - key, - filename="neutral density", - figheader=figheader, - filetype=filetype, - phantom_profile=phantom_profiles, - logscale=True, - ) - - corner.corner(post_sample, labels=param_names) - plt.savefig(figheader + "posterior" + filetype) - - corner.corner(prior_sample, labels=param_names) - plt.savefig(figheader + "prior" + filetype) - plt.close("all") + corner.corner(prior_sample[t_idx,], labels=param_names) + plt.savefig(figheader + "prior" + filetype) + plt.close("all") if __name__ == "__main__": From 2c6634a1a80faefa67de86b175608368fc89f3e6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 09:41:33 +0100 Subject: [PATCH 208/288] renamed some nodes and removed TIME_BINS --- indica/writers/bda_tree.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 293b3741..25453c96 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -10,10 +10,9 @@ def bda(): nodes = { "TIME": ("NUMERIC", "time vector of optimisation, s"), - "TIME_BINS": ("NUMERIC", "time vector used for binning, s"), "INPUT": { "BURN_FRAC": ("NUMERIC", "Burn in fraction for chains"), - "ITER": ("NUMERIC", "Iterations of optimiser"), + "ITER": ("NUMERIC", "Maximum iterations of optimiser"), "PARAM_NAMES": ("TEXT", "Names of parameters optimised"), "OPT_QUANTITY": ("TEXT", "Names of quantities optimised"), "MODEL_KWARGS": ("TEXT", "Model key word arguments"), @@ -22,7 +21,6 @@ def bda(): "TSTART": ("NUMERIC", "Start of time vector, s"), "TEND": ("NUMERIC", "End of time vector, s"), "DT": ("NUMERIC", "Distance between time points, s"), - "TSAMPLE": ("NUMERIC", "Sample time, s"), "IMPURITIES": ("TEXT", "Names of impurity elements"), "MAIN_ION": ("TEXT", "Name of main ion"), }, @@ -67,7 +65,7 @@ def bda(): "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), }, "PROFILE_STAT": { - "SAMPLES": ("NUMERIC", "Numerical index of the optimisation samples"), + "SAMPLE_IDX": ("NUMERIC", "Index of the optimisation samples"), "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), "NE": ("SIGNAL", "Electron density, m^-3"), "NI": ("SIGNAL", "Ion density, m^-3"), @@ -138,10 +136,10 @@ def bda(): }, "OPTIMISATION": { "ACCEPT_FRAC": ("NUMERIC", "Fraction of samples accepted by optimiser"), - "AUTO_CORR": ("NUMERIC", "Auto-correlation (iteration nwalker)"), - "POST_SAMPLE": ("NUMERIC", "Posterior probability samples (sample)"), + "AUTO_CORR": ("NUMERIC", "Auto-correlation"), + "POST_SAMPLE": ("NUMERIC", "Posterior probability samples"), "PRIOR_SAMPLE": ("NUMERIC", "Prior samples"), - "GELMANRUBIN": ("NUMERIC", "Gelmin-Rubin convergence diagnostic"), + "GELMAN_RUBIN": ("NUMERIC", "Gelmin-Rubin convergence diagnostic"), }, } return nodes @@ -192,7 +190,7 @@ def create_nodes( } for diag_name in diag_names } - model_nodes["SAMPLES"] = ("NUMERIC", "index of samples taken from optimisation") + model_nodes["SAMPLE_IDX"] = ("NUMERIC", "Index of the optimisation samples") bda_nodes["MODEL_DATA"] = model_nodes bda_nodes["DIAG_DATA"] = diag_nodes bda_nodes["INPUT"]["WORKFLOW"] = workflow_nodes From 65060e958e10f3a0907a24340603921f339168a5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 09:47:16 +0100 Subject: [PATCH 209/288] renamed bayes_workflow_example -> bayes_workflow --- .../{bayes_workflow_example.py => bayes_workflow.py} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename indica/workflows/{bayes_workflow_example.py => bayes_workflow.py} (99%) diff --git a/indica/workflows/bayes_workflow_example.py b/indica/workflows/bayes_workflow.py similarity index 99% rename from indica/workflows/bayes_workflow_example.py rename to indica/workflows/bayes_workflow.py index 8fbdef68..7c950668 100644 --- a/indica/workflows/bayes_workflow_example.py +++ b/indica/workflows/bayes_workflow.py @@ -892,7 +892,7 @@ def format_results(self): return results -class BayesWorkflowExample(AbstractBayesWorkflow): +class BayesWorkflow(AbstractBayesWorkflow): def __init__( self, bayes_settings: BayesSettings, @@ -1064,9 +1064,9 @@ def __call__( priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) - workflow = BayesWorkflowExample(tstart=tstart, tend=tend, dt=dt, - bayes_settings=bayes_settings, data_context=data_context, - optimiser_context=optimiser_context, - plasma_context=plasma_context, model_context=model_context) + workflow = BayesWorkflow(tstart=tstart, tend=tend, dt=dt, + bayes_settings=bayes_settings, data_context=data_context, + optimiser_context=optimiser_context, + plasma_context=plasma_context, model_context=model_context) workflow(pulse_to_write=25000000, run="RUN01", mds_write=True, plot=True, filepath="./results/test_moments/") From d81581fd9bd03e497a570cf75056d8d91e515a21 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 10:14:41 +0100 Subject: [PATCH 210/288] missing self. --- indica/workflows/bayes_workflow.py | 2 +- indica/workflows/{batch_bda.py => bda_run.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename indica/workflows/{batch_bda.py => bda_run.py} (100%) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 7c950668..f43a393d 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -608,7 +608,7 @@ class ExpData(DataContext): def read_data(self, ): self.reader = ReadST40(self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt, ) self.reader(self.diagnostics, revisions=self.reader_settings.revisions, filters=self.reader_settings.filters) - missing_keys = set(diagnostics) - set(self.reader.binned_data.keys()) + missing_keys = set(self.diagnostics) - set(self.reader.binned_data.keys()) if len(missing_keys) > 0: raise ValueError(f"missing data: {missing_keys}") self.equilibrium = self.reader.equilibrium diff --git a/indica/workflows/batch_bda.py b/indica/workflows/bda_run.py similarity index 100% rename from indica/workflows/batch_bda.py rename to indica/workflows/bda_run.py From bfd60275639a71a6cb900b518a353e6212ab5456 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 10:15:20 +0100 Subject: [PATCH 211/288] missing self. --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index f43a393d..c32d2454 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -456,7 +456,7 @@ class ModelContext: def update_model_kwargs(self, data: dict): if self.model_kwargs is None: self.model_kwargs = {} - for diag in diagnostics: + for diag in self.diagnostics: if diag not in self.model_kwargs.keys(): self.model_kwargs[diag] = {} From 1b055800edc79dbf324d1a94ebf6f9aa098af865 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 10:16:28 +0100 Subject: [PATCH 212/288] removed debug printing from sample_with_moments --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index c32d2454..16194686 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -172,7 +172,7 @@ ] -def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample=10, stopping_factor=10, debug=True): +def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample=10, stopping_factor=10, debug=False): # TODO: Compare old_chain to new_chain: if moments are different then keep going / convergence diagnostics here autocorr = np.ones(shape=(iterations, n_params)) * np.nan From 1955db211ec73305623c60bd55c2e67fc04bcdd1 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 10:16:55 +0100 Subject: [PATCH 213/288] renamed batch_bda -> bda_run --- indica/workflows/bda_run.py | 276 +++++++++--------------------------- 1 file changed, 68 insertions(+), 208 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index ef698ec1..3cea8645 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -1,26 +1,5 @@ -from indica.workflows.bayes_workflow_example import BayesWorkflowExample, DEFAULT_PRIORS, DEFAULT_PROFILE_PARAMS - -PARAMS_SET_TS =[ - # "Ne_prof.y1", - # "Ne_prof.y0", - # "Ne_prof.peaking", - # "Ne_prof.wcenter", - # "Ne_prof.wped", - # "Nimp_prof.y1", - "Nimp_prof.y0", - "Nimp_prof.wcenter", - # "Nimp_prof.wped", - "Nimp_prof.peaking", - # "Te_prof.y0", - # "Te_prof.wped", - # "Te_prof.wcenter", - # "Te_prof.peaking", - "Ti_prof.y0", - # # "Ti_prof.wped", - "Ti_prof.wcenter", - "Ti_prof.peaking", - "xrcs.pixel_offset", - ] +from indica.workflows.bayes_workflow import BayesWorkflow, DEFAULT_PRIORS, DEFAULT_PROFILE_PARAMS, BayesSettings, ReaderSettings, \ + ExpData, MockData, PhantomData, PlasmaSettings, PlasmaContext, ModelContext, EmceeOptimiser, OptimiserEmceeSettings PARAMS_DEFAULT =[ "Ne_prof.y1", @@ -44,29 +23,6 @@ "xrcs.pixel_offset", ] -PARAMS_SET_ALL =[ - # "Ne_prof.y1", - # "Ne_prof.y0", - # "Ne_prof.peaking", - # "Ne_prof.wcenter", - # "Ne_prof.wped", - # "Nimp_prof.y1", - # "Nimp_prof.y0", - # "Nimp_prof.wcenter", - # "Nimp_prof.wped", - # "Nimp_prof.peaking", - # "Te_prof.y0", - # "Te_prof.wped", - # "Te_prof.wcenter", - # "Te_prof.peaking", - # "Ti_prof.y0", - # "Ti_prof.wped", - # "Ti_prof.wcenter", - # "Ti_prof.peaking", - "xrcs.pixel_offset", - ] - - DIAG_DEFAULT = [ "xrcs", "ts", @@ -94,16 +50,6 @@ # "smmh1", ] - -DIAG_SET_TS = [ - "xrcs", - # "ts", - "efit", - "cxff_pi", - "cxff_tws_c" - # "smmh1", - ] - OPT_DEFAULT = [ "xrcs.spectra", "ts.ne", @@ -114,174 +60,88 @@ # "smmh1.ne" ] -OPT_SET_TS = [ - "xrcs.spectra", - # "ts.ne", - # "ts.te", - "efit.wp", - "cxff_pi.ti", - "cxff_tws_c.ti", - # "smmh1.ne" - ] -def pulse_example(pulse, diagnostics, param_names, opt_quantity, t_opt, iterations=100, run="RUN01", - fast_particles=False, astra_run="RUN602", astra_pulse_range=13000000, - astra_equilibrium=False, efit_revision=0, set_ts_profiles=False, set_all_profiles=True, - astra_wp=False, **kwargs): +def bda_run(pulse, diagnostics, param_names, opt_quantity, + tstart=0.02, tend=0.03, dt=0.01, revisions = {}, filters={}, + iterations=500, nwalkers=50, mds_write=False, plot=False, + run="RUN01", **kwargs): + + # BlackBoxSettings + bayes_settings = BayesSettings(diagnostics=diagnostics, param_names=param_names, + opt_quantity=opt_quantity, priors=DEFAULT_PRIORS, ) - workflow = BayesWorkflowExample( - pulse=pulse, - diagnostics=diagnostics, - param_names=param_names, - opt_quantity=opt_quantity, - priors=DEFAULT_PRIORS, - profile_params=DEFAULT_PROFILE_PARAMS, - phantoms=False, - fast_particles=fast_particles, - tstart=0.06, - tend=0.10, - dt=0.005, - astra_run=astra_run, - astra_pulse_range=astra_pulse_range, - astra_equilibrium=astra_equilibrium, - efit_revision=efit_revision, - set_ts_profiles = set_ts_profiles, - set_all_profiles=set_all_profiles, - astra_wp = astra_wp, - ) + data_settings = ReaderSettings(filters=filters, + revisions=revisions) - workflow.setup_plasma( - tsample=t_opt, - # n_rad=50 - ) - workflow.setup_opt_data(phantoms=workflow.phantoms) - workflow.setup_optimiser(nwalkers=40, sample_method="high_density", model_kwargs=workflow.model_kwargs, nsamples=100) - results = workflow( - filepath=f"./results/{workflow.pulse}.{run}/", - pulse_to_write=25000000 + workflow.pulse, - run=run, - mds_write=False, - plot=True, - burn_frac=0.20, - iterations=iterations, - ) + data_context = ExpData(pulse=pulse, diagnostics=diagnostics, + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + data_context.read_data() -# (11089, DIAG_DEFAULT_CHERS, PARAMS_DEFAULT, [ - # "xrcs.spectra", - # "ts.ne", - # "ts.te", - # "efit.wp", - # # "cxff_pi.ti", - # "cxff_tws_c.ti", - # # "smmh1.ne" - # ], 0.100, - # dict(run="RUN01", fast_particles=True, astra_run="RUN601", astra_pulse_range=13000000, astra_equilibrium=False, - # efit_revision=2)), + plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), + n_rad=20) + plasma_context = PlasmaContext(plasma_settings=plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) + model_init_kwargs = { + "cxff_pi": {"element": "ar"}, + "cxff_tws_c": {"element": "c"}, + "xrcs": { + "window_masks": [slice(0.394, 0.396)], + }, + } + + model_context = ModelContext(diagnostics=diagnostics, + plasma_context=plasma_context, + equilibrium=data_context.equilibrium, + transforms=data_context.transforms, + model_kwargs=model_init_kwargs, + ) + + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=nwalkers, iterations=iterations, + sample_method="high_density", starting_samples=100, burn_frac=0.20, + stopping_criterion="mode", stopping_criterion_factor=10, + priors=bayes_settings.priors) + optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) + + workflow = BayesWorkflow(tstart=tstart, tend=tend, dt=dt, + bayes_settings=bayes_settings, data_context=data_context, + optimiser_context=optimiser_context, + plasma_context=plasma_context, model_context=model_context) + + workflow(pulse_to_write=25000000+pulse, run=run, mds_write=mds_write, plot=plot, filepath=f"./results/{pulse}/") if __name__ == "__main__": pulse_info = [ - # (11211, DIAG_DEFAULT, PARAMS_DEFAULT, OPT_DEFAULT, 0.084), - # (11215, DIAG_DEFAULT_CHERS, PARAMS_DEFAULT, [ - # "xrcs.spectra", - # "ts.ne", - # "ts.te", - # "efit.wp", - # # "cxff_pi.ti", - # "cxff_tws_c.ti", - # # "smmh1.ne" - # ], 0.070, dict(fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), - # - # # (11224, DIAG_DEFAULT, PARAMS_DEFAULT, OPT_DEFAULT, 0.090), - # (11225, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ - # "xrcs.spectra", - # "ts.ne", - # "ts.te", - # "efit.wp", - # "cxff_pi.ti", - # # "cxff_tws_c.ti", - # # "smmh1.ne" - # ], 0.090, dict(efit_revision=2, fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), - # - # # (11226, DIAG_DEFAULT, PARAMS_DEFAULT, OPT_DEFAULT, 0.070), - # (11227, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ - # "xrcs.spectra", - # "ts.ne", - # "ts.te", - # "efit.wp", - # "cxff_pi.ti", - # # "cxff_tws_c.ti", - # # "smmh1.ne" - # ], 0.070, dict(fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), - # - # (11228, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ - # "xrcs.spectra", - # "ts.ne", - # "ts.te", - # "efit.wp", - # "cxff_pi.ti", - # # "cxff_tws_c.ti", - # # "smmh1.ne" - # ], 0.080, dict(run="RUN02", fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), - # (11238, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ - # "xrcs.spectra", - # "ts.ne", - # "ts.te", - # "efit.wp", - # "cxff_pi.ti", - # # "cxff_tws_c.ti", - # # "smmh1.ne" - # ], 0.075, dict(efit_revision=2, fast_particles=True, astra_run="RUN603", astra_pulse_range=13000000, astra_equilibrium=True, astra_wp=True)), - # - # (11312, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ - # "xrcs.spectra", - # "ts.ne", - # "ts.te", - # "efit.wp", - # "cxff_pi.ti", - # # "cxff_tws_c.ti", - # # "smmh1.ne" - # ], 0.080, dict(run="RUN01", fast_particles=True, astra_run="RUN14", astra_pulse_range=33000000, astra_equilibrium=True, set_ts_profiles=False, set_all_profiles=False, astra_wp=True)), - # - # (11314, DIAG_DEFAULT_PI, PARAMS_DEFAULT, [ - # "xrcs.spectra", - # "ts.ne", - # "ts.te", - # "efit.wp", - # "cxff_pi.ti", - # # "cxff_tws_c.ti", - # # "smmh1.ne" - # ], 0.080, dict(run="RUN01", fast_particles=True, astra_run="RUN3", astra_pulse_range=33000000, astra_equilibrium=True, set_ts_profiles=False, astra_wp=True)), - (11312, DIAG_SET_TS, PARAMS_SET_TS, [ - "xrcs.spectra", + [(11366, + ["ts"], + ["Te_prof.y0", + "Te_prof.peaking", + "Te_prof.wped", + "Te_prof.wcenter", + ], + [ + # "xrcs.spectra", # "ts.ne", - # "ts.te", + "ts.te", # "efit.wp", - "cxff_pi.ti", + # "cxff_pi.ti", # "cxff_tws_c.ti", # "smmh1.ne" - ], 0.080, - dict(run="TEST_TI", fast_particles=True, astra_run="RUN14", astra_pulse_range=33000000, astra_equilibrium=True, - set_ts_profiles=True, set_all_profiles=False, astra_wp=True)), + ]), + dict( + tstart=0.07, + tend=0.08, + dt=0.01, + iterations=500, + nwalkers=50, + mds_write=True, + plot=True, + run="RUN01", + )], - # Tree doesnt exist - ## (11317, 0.080, dict(run="RUN01", fast_particles=True, astra_run="RUN11", astra_pulse_range=33000000, astra_equilibrium=True, set_ts_profiles=True, astra_wp=True)), ] - # for info in pulse_info: - # if len(info) < 6: - # info = list(info) - # info.append(dict()) - # info = tuple(info) - # try: - # pulse_example(*info[0:5], iterations=2, **info[5]) - # except Exception as e: - # print(f"pulse: {info[0]}") - # print(repr(e)) - # continue - for info in pulse_info: - pulse_example(*info[0:5], iterations=2000, **info[5]) \ No newline at end of file + bda_run(*info[0], **info[1]) \ No newline at end of file From 89393c9fac20ee4b4dea4b9111c9aea93710a6aa Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 10:23:20 +0100 Subject: [PATCH 214/288] debug printing for stopping condition is now an optional input --- indica/workflows/bayes_workflow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 16194686..81933580 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -722,6 +722,7 @@ class OptimiserEmceeSettings: starting_samples: int = 100 stopping_criterion: str = "mode" stopping_criterion_factor: float = 10 + stopping_criteria_debug: bool = False @dataclass @@ -852,7 +853,8 @@ def run(self, ): self.optimiser_settings.iterations, self.optimiser_settings.param_names.__len__(), auto_sample=10, - stopping_factor=self.optimiser_settings.stopping_criterion_factor + stopping_factor=self.optimiser_settings.stopping_criterion_factor, + debug = self.optimiser_settings.stopping_criteria_debug ) else: raise ValueError(f"Stopping criterion: {self.optimiser_settings.stopping_criterion} not recognised") From 256e41145c513350ee50ea79cbd72b17212757ff Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 10:30:52 +0100 Subject: [PATCH 215/288] moved back to flat delta moment stopping condition --- indica/workflows/bayes_workflow.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 81933580..de505c40 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -172,7 +172,7 @@ ] -def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample=10, stopping_factor=10, debug=False): +def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample=10, stopping_factor=0.01, debug=False): # TODO: Compare old_chain to new_chain: if moments are different then keep going / convergence diagnostics here autocorr = np.ones(shape=(iterations, n_params)) * np.nan @@ -194,14 +194,12 @@ def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample new_mean = dist_stats.mean dmean = np.abs(new_mean - old_mean) - nsamples = sampler.flatchain.shape[0] - - dmean_normed = dmean / old_mean * nsamples + rel_dmean = dmean / old_mean if debug: print("") - print(f"dmean_normed: {dmean_normed.max()}") - if dmean_normed.max() < stopping_factor: + print(f"rel_dmean: {rel_dmean.max()}") + if rel_dmean.max() < stopping_factor: if success_flag: break else: @@ -721,7 +719,7 @@ class OptimiserEmceeSettings: sample_method: str = "random" starting_samples: int = 100 stopping_criterion: str = "mode" - stopping_criterion_factor: float = 10 + stopping_criterion_factor: float = 0.01 stopping_criteria_debug: bool = False @@ -1062,7 +1060,7 @@ def __call__( optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=500, sample_method="high_density", starting_samples=100, burn_frac=0.20, - stopping_criterion="mode", stopping_criterion_factor=10, + stopping_criterion="mode", stopping_criterion_factor=0.01, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) From 4228a4ee7e2133f86b51bf8e67c6e04f398981e7 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 10:37:46 +0100 Subject: [PATCH 216/288] default to plotting to test --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index de505c40..ec8dc363 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -1069,4 +1069,4 @@ def __call__( optimiser_context=optimiser_context, plasma_context=plasma_context, model_context=model_context) - workflow(pulse_to_write=25000000, run="RUN01", mds_write=True, plot=True, filepath="./results/test_moments/") + workflow(pulse_to_write=25000000, run="RUN01", mds_write=True, plot=True, filepath="./results/test/") From 4899aeff42a6fbd24df45f931be2d5f6555c53ab Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 10:52:19 +0100 Subject: [PATCH 217/288] comments --- indica/workflows/bayes_workflow.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index ec8dc363..67aa1304 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -624,6 +624,9 @@ def _fail_strategy(self): raise ValueError("Data strategy: Fail") def _process_data(self): + """ + Returns flattened bin data and adds custom errors + """ opt_data = flatdict.FlatDict(self.binned_data, ".") if "xrcs.spectra" in opt_data.keys(): From f0590a6d6187c3a244f32dc217ab6ee0475e67b1 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 17 Oct 2023 13:47:39 +0100 Subject: [PATCH 218/288] moved sample_from_priors to external function --- indica/workflows/bayes_workflow.py | 61 +++++++++++++++--------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 67aa1304..58b507f0 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -747,6 +747,35 @@ def run(self): results = None return results +def sample_from_priors(param_names: list, priors: dict, size=10): + # Use priors to generate samples + for name in param_names: + if name in priors.keys(): + if hasattr(priors[name], "rvs"): + continue + else: + raise TypeError(f"prior object {name} missing rvs method") + else: + raise ValueError(f"Missing prior for {name}") + + # Throw out samples that don't meet conditional priors and redraw + samples = np.empty((param_names.__len__(), 0)) + while samples.size < param_names.__len__() * size: + # Some mangling of dictionaries so _ln_prior works + # Increase size * n if too slow / looping too much + new_sample = { + name: priors[name].rvs(size=size * 2) for name in param_names + } + _ln_prior = ln_prior(priors, new_sample) + # Convert from dictionary of arrays -> array, + # then filtering out where ln_prior is -infinity + accepted_samples = np.array(list(new_sample.values()))[ + :, _ln_prior != -np.inf + ] + samples = np.append(samples, accepted_samples, axis=1) + samples = samples[:, 0:size] + return samples.transpose() + @dataclass class EmceeOptimiser(OptimiserContext): @@ -777,7 +806,7 @@ def sample_start_points(self, ): ) elif self.optimiser_settings.sample_method == "random": - self.start_points = self.sample_from_priors( + self.start_points = sample_from_priors( param_names=self.optimiser_settings.param_names, priors=self.optimiser_settings.priors, size=self.optimiser_settings.nwalkers @@ -786,41 +815,13 @@ def sample_start_points(self, ): raise ValueError(f"Sample method: {self.optimiser_settings.sample_method} " f"not recognised, Defaulting to random sampling") - def sample_from_priors(self, param_names: list, priors: dict, size=10): - # Use priors to generate samples - for name in param_names: - if name in priors.keys(): - if hasattr(priors[name], "rvs"): - continue - else: - raise TypeError(f"prior object {name} missing rvs method") - else: - raise ValueError(f"Missing prior for {name}") - - # Throw out samples that don't meet conditional priors and redraw - samples = np.empty((param_names.__len__(), 0)) - while samples.size < param_names.__len__() * size: - # Some mangling of dictionaries so _ln_prior works - # Increase size * n if too slow / looping too much - new_sample = { - name: priors[name].rvs(size=size * 2) for name in param_names - } - _ln_prior = ln_prior(priors, new_sample) - # Convert from dictionary of arrays -> array, - # then filtering out where ln_prior is -infinity - accepted_samples = np.array(list(new_sample.values()))[ - :, _ln_prior != -np.inf - ] - samples = np.append(samples, accepted_samples, axis=1) - samples = samples[:, 0:size] - return samples.transpose() def sample_from_high_density_region( self, param_names: list, priors: dict, optimiser, nwalkers: int, nsamples=100 ): # TODO: remove repeated code - start_points = self.sample_from_priors(param_names, priors, size=nsamples) + start_points = sample_from_priors(param_names, priors, size=nsamples) ln_prob, _ = optimiser.compute_log_prob(start_points) num_best_points = 3 From bb2e8e91be37448557589e5ba65365f5b74d3b80 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 19 Oct 2023 11:39:47 +0100 Subject: [PATCH 219/288] moved checking of prior keys from sample_from_priors to BayesSettings dataclass --- indica/workflows/bayes_workflow.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 58b507f0..30d4b62c 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -282,6 +282,16 @@ def __post_init__(self): if missing_quantities: raise ValueError(f"{missing_quantities} missing the relevant diagnostic") + # check all priors are defined + for name in self.param_names: + if name in self.priors.keys(): + if hasattr(self.priors[name], "rvs"): + continue + else: + raise TypeError(f"prior object {name} missing rvs method") + else: + raise ValueError(f"Missing prior for {name}") + @dataclass class PlasmaSettings: @@ -748,16 +758,6 @@ def run(self): return results def sample_from_priors(param_names: list, priors: dict, size=10): - # Use priors to generate samples - for name in param_names: - if name in priors.keys(): - if hasattr(priors[name], "rvs"): - continue - else: - raise TypeError(f"prior object {name} missing rvs method") - else: - raise ValueError(f"Missing prior for {name}") - # Throw out samples that don't meet conditional priors and redraw samples = np.empty((param_names.__len__(), 0)) while samples.size < param_names.__len__() * size: From 55e01f83ac3136c372b4d0387352e58988cc22e4 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 13:09:46 +0100 Subject: [PATCH 220/288] _build_bckc now creates nested dictionary instead of flat --- indica/bayesblackbox.py | 4 +- indica/workflows/bayes_workflow.py | 153 +++++++++++++---------------- 2 files changed, 72 insertions(+), 85 deletions(-) diff --git a/indica/bayesblackbox.py b/indica/bayesblackbox.py index 7b74f6e1..3fd90bc6 100644 --- a/indica/bayesblackbox.py +++ b/indica/bayesblackbox.py @@ -1,6 +1,6 @@ from copy import deepcopy import warnings - +from flatdict import FlatDict import numpy as np from scipy.stats import uniform np.seterr(all="ignore") @@ -134,7 +134,7 @@ def ln_posterior(self, parameters: dict, **kwargs): self.plasma_context.update_profiles(parameters) plasma_attributes = self.plasma_context.return_plasma_attrs() - self.bckc = self.model_context._build_bckc(parameters, **kwargs) # model calls + self.bckc = FlatDict(self.model_context._build_bckc(parameters, **kwargs), ".") # model calls _ln_likelihood = self.ln_likelihood() # compare results to data ln_posterior = _ln_likelihood + _ln_prior diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 30d4b62c..e3a92af8 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -271,6 +271,7 @@ class BayesSettings: param_names: list = field(default_factory=lambda: OPTIMISED_PARAMS) opt_quantity: list = field(default_factory=lambda: OPTIMISED_QUANTITY) priors: dict = field(default_factory=lambda: DEFAULT_PRIORS) + percent_error: float = 0.10 """ TODO: default methods / getter + setters @@ -339,7 +340,7 @@ def update_profiles(self, params: dict): self.plasma.update_profiles(params) - def time_iterator(self): # TODO: Why is this being called anytime plasma attributes are accessed? + def time_iterator(self): print("resetting time iterator") return iter(self.plasma.t) @@ -522,7 +523,7 @@ def init_models(self, ): return self.models - def _build_bckc(self, params: dict, **kwargs): + def _build_bckc(self, params: dict = None, **kwargs): """ Parameters ---------- @@ -531,13 +532,14 @@ def _build_bckc(self, params: dict, **kwargs): Returns ------- - bckc of results + nested bckc of results """ # Float128 since rounding of small numbers causes problems # when initial results are bad fits # model_data = self.bckc[key].astype("float128") - + if params is None: + params = {} self.bckc: dict = {} for model_name, model in self.models.items(): # removes "model.name." from params and kwargs then passes them to model @@ -558,10 +560,8 @@ def _build_bckc(self, params: dict, **kwargs): **_model_settings, } # combine dictionaries _bckc = model(**_model_kwargs) - _model_bckc = { - f"{model.name}.{value_name}": value - for value_name, value in _bckc.items() - } # prepend model name to bckc + _model_bckc = {model.name: {value_name: value for value_name, value in _bckc.items()}} + # prepend model name to bckc self.bckc = dict(self.bckc, **_model_bckc) return self.bckc @@ -579,6 +579,7 @@ class DataContext(ABC): tstart: float tend: float dt: float + diagnostics: list transforms = None equilibrium = None phantoms = False @@ -590,22 +591,27 @@ def read_data(self, ): self.raw_data = None self.binned_data = None - def check_if_data_present(self, data_strategy: Callable): + @abstractmethod + def data_strategy(self): + return None + + def _check_if_data_present(self, data_strategy: Callable = lambda: None): if not self.binned_data: print("Data not given: using data strategy") self.binned_data = data_strategy() + def pre_process_data(self, model_callable: Callable): + self.model_callable = model_callable # TODO: handle this dependency (phantom data) some other way? + self._check_if_data_present(self.data_strategy) + @abstractmethod - def process_data(self, model_context: ModelContext): - self.model_context = model_context # TODO: handle this dependency (phantom data) some other way - self.check_if_data_present(lambda: {}) + def process_data(self, model_callable: Callable): + self.pre_process_data(model_context) self.opt_data = flatdict.FlatDict(self.binned_data) @dataclass class ExpData(DataContext): - pulse: int - diagnostics: list phantoms = False """ @@ -625,20 +631,12 @@ def read_data(self, ): self.raw_data = self.reader.raw_data self.binned_data = self.reader.binned_data - def process_data(self, model_callables: Callable): - self.model_callables = model_callables - self.check_if_data_present(self._fail_strategy) - self.opt_data = self._process_data() - - def _fail_strategy(self): + def data_strategy(self): raise ValueError("Data strategy: Fail") - def _process_data(self): - """ - Returns flattened bin data and adds custom errors - """ + def process_data(self, model_callable: Callable): + self.pre_process_data(model_callable) opt_data = flatdict.FlatDict(self.binned_data, ".") - if "xrcs.spectra" in opt_data.keys(): background = opt_data["xrcs.spectra"].where( (opt_data["xrcs.spectra"].wavelength < 0.392) @@ -653,48 +651,10 @@ def _process_data(self): if "ts.te" in opt_data.keys(): opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 - - return opt_data - - -@dataclass -class MockData(DataContext): - pulse = None - diagnostic_transforms: dict = field(default_factory=lambda: {}) - model_call_params: dict = field(default_factory=lambda: {}) - phantoms = True - - def read_data(self): - # Used with phantom data for purposes of tests - print("Reading mock equilibrium / transforms") - self.equilibrium = fake_equilibrium( - tstart, - tend, - dt, - ) - self.transforms = self.diagnostic_transforms - self.binned_data: dict = {} - self.raw_data: dict = {} - - def process_data(self, model_callables: Callable): - self.model_callables = model_callables - self.check_if_data_present(self._gen_data_strategy) - - def _gen_data_strategy(self): - print("Data strategy: generating from model bckc") - self.binned_data = self._process_data() - - def _process_data(self): - binned_data = self.model_callables() - self.opt_data = flatdict.FlatDict(binned_data, ".") - return binned_data - + self.opt_data = opt_data @dataclass class PhantomData(DataContext): - pulse: int - diagnostics: list - model_call_kwargs: dict = field(default_factory=lambda: {}) phantoms = True def read_data(self, ): @@ -708,18 +668,32 @@ def read_data(self, ): self.raw_data = {} self.binned_data = {} - def process_data(self, model_callables: Callable): - self.model_callables = model_callables - self.check_if_data_present(self._gen_data_strategy) + def data_strategy(self): + print("Data strategy: Phantom data") + return self.model_callable() - def _gen_data_strategy(self): - print("Data strategy: generating from model bckc") - self.binned_data = self._process_data() + def process_data(self, model_callable: Callable): + self.pre_process_data(model_callable) + self.opt_data = flatdict.FlatDict(self.binned_data, ".") + +@dataclass +class MockData(PhantomData): + diagnostic_transforms: dict = field(default_factory=lambda: {}) + + def read_data(self): + print("Reading mock equilibrium / transforms") + self.equilibrium = fake_equilibrium( + tstart, + tend, + dt, + ) + missing_transforms = list(set(diagnostics).difference(self.diagnostic_transforms.keys())) + if missing_transforms: + raise ValueError(f"Missing transforms: {missing_transforms}") - def _process_data(self): - binned_data = self.model_callables() - self.opt_data = flatdict.FlatDict(binned_data, ".") - return binned_data + self.transforms = self.diagnostic_transforms + self.binned_data: dict = {} + self.raw_data: dict = {} @dataclass @@ -758,6 +732,12 @@ def run(self): return results def sample_from_priors(param_names: list, priors: dict, size=10): + """ + TODO: may be able to remove param_names from here and at some point earlier remove priors that aren't used + then loop over remaining priors while handling conditional priors somehow... + The order of samples may need to be checked / reordered at some point then + """ + # Throw out samples that don't meet conditional priors and redraw samples = np.empty((param_names.__len__(), 0)) while samples.size < param_names.__len__() * size: @@ -881,7 +861,7 @@ def format_results(self): for blob_name in blob_names } results["accept_frac"] = self.optimiser.acceptance_fraction.sum() - results["prior_sample"] = self.sample_from_priors( + results["prior_sample"] = sample_from_priors( self.optimiser_settings.param_names, self.optimiser_settings.priors, size=int(1e4) ) @@ -927,14 +907,15 @@ def __init__( self.model_context.update_model_kwargs(self.data_context.binned_data) self.model_context.init_models() - self.model_call_kwargs = {"xrcs.pixel_offset": 4.0} - self.data_context.process_data(partial(self.model_context._build_bckc, self.model_call_kwargs)) + self.model_call_kwargs = {"xrcs.pixel_offset": -4.0} + self.data_context.process_data(partial(self.model_context._build_bckc, **self.model_call_kwargs)) self.blackbox = BayesBlackBox(data=self.data_context.opt_data, plasma_context=self.plasma_context, model_context=self.model_context, quant_to_optimise=self.bayes_settings.opt_quantity, - priors=self.bayes_settings.priors) + priors=self.bayes_settings.priors, + percent_error=self.bayes_settings.percent_error) self.optimiser_context.init_optimiser(self.blackbox.ln_posterior, model_kwargs=self.model_call_kwargs) @@ -1004,15 +985,16 @@ def __call__( dt = 0.01 diagnostics = [ - # "xrcs", - # "efit", + "xrcs", + "efit", # "smmh1", # "cxff_pi", + "cxff_tws_c", "ts", ] # diagnostic_quantities opt_quant = [ - # "xrcs.spectra", + "xrcs.spectra", # "efit.wp", "ts.te" ] @@ -1038,11 +1020,16 @@ def __call__( # "cxff_pi": pi_transform_example(5), # "ts": ts_transform_example(11), } # data_context = MockData(pulse=None, diagnostic_transforms=mock_transforms, ) + # + # data_context = ExpData(pulse=pulse, diagnostics=diagnostics, + # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + # data_context.read_data() data_context = ExpData(pulse=pulse, diagnostics=diagnostics, - tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context.read_data() + plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), n_rad=20) plasma_context = PlasmaContext(plasma_settings=plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) @@ -1062,7 +1049,7 @@ def __call__( model_kwargs=model_init_kwargs, ) - optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=500, + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=5, sample_method="high_density", starting_samples=100, burn_frac=0.20, stopping_criterion="mode", stopping_criterion_factor=0.01, priors=bayes_settings.priors) From 5219422e68ad5343c1a9dbcafbc10df379a13f87 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 13:10:14 +0100 Subject: [PATCH 221/288] filtering now works on spectra with t dim --- indica/models/helike_spectroscopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/helike_spectroscopy.py b/indica/models/helike_spectroscopy.py index 7e3fc5db..bc0fbc39 100644 --- a/indica/models/helike_spectroscopy.py +++ b/indica/models/helike_spectroscopy.py @@ -216,7 +216,7 @@ def _make_spectra(self, calc_rho: bool = False): measured_spectra = measured_spectra.assign_coords( {"wavelength": self.spectra.wavelength} ) - measured_spectra[measured_spectra==0] = np.nan + measured_spectra = xr.where(measured_spectra == 0, np.nan, measured_spectra) self.measured_spectra = measured_spectra.sortby("wavelength") self.spectra_los = self.los_transform.along_los From 153536bdbb57fe6d81519b017a102dc97f5653e9 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 13:15:55 +0100 Subject: [PATCH 222/288] renamed bayes_settings -> blackbox_settings --- indica/workflows/abstract_bayes_workflow.py | 20 ++-- indica/workflows/bayes_workflow.py | 111 ++++++++++---------- 2 files changed, 63 insertions(+), 68 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 8c929ac0..d2e1be2a 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -4,12 +4,6 @@ import pickle import numpy as np -import pandas as pd -import xarray as xr - -from indica.equilibrium import fake_equilibrium -from indica.readers.read_st40 import ReadST40 -import scipy.stats as stats class AbstractBayesWorkflow(ABC): @abstractmethod @@ -41,7 +35,7 @@ def _build_inputs_dict(self): """ result = {} - quant_list = [item.split(".") for item in self.bayes_settings.opt_quantity] + quant_list = [item.split(".") for item in self.blackbox_settings.opt_quantity] result["TIME"] = self.plasma_context.plasma.t @@ -56,8 +50,8 @@ def _build_inputs_dict(self): "ITER": self.optimiser_context.optimiser_settings.iterations, "NWALKERS": self.optimiser_context.optimiser_settings.nwalkers, "MODEL_KWARGS": self.model_call_kwargs, - "OPT_QUANTITY": self.bayes_settings.opt_quantity, - "PARAM_NAMES": self.bayes_settings.param_names, + "OPT_QUANTITY": self.blackbox_settings.opt_quantity, + "PARAM_NAMES": self.blackbox_settings.param_names, "PULSE": self.data_context.pulse, "IMPURITIES": self.plasma_context.plasma_settings.impurities, "MAIN_ION": self.plasma_context.plasma_settings.main_ion, @@ -74,7 +68,7 @@ def _build_inputs_dict(self): ), "RUN": "PLACEHOLDER", } - for diag_name in self.bayes_settings.diagnostics + for diag_name in self.blackbox_settings.diagnostics } result["DIAG_DATA"] = { @@ -83,7 +77,7 @@ def _build_inputs_dict(self): for quantity in quant_list if quantity[0] == diag_name } - for diag_name in self.bayes_settings.diagnostics + for diag_name in self.blackbox_settings.diagnostics } return result @@ -91,7 +85,7 @@ def _build_inputs_dict(self): def _build_result_dict(self, ): result = {} - quant_list = [item.split(".") for item in self.bayes_settings.opt_quantity] + quant_list = [item.split(".") for item in self.blackbox_settings.opt_quantity] result["MODEL_DATA"] = { diag_name.upper(): { @@ -99,7 +93,7 @@ def _build_result_dict(self, ): for quantity in quant_list if quantity[0] == diag_name } - for diag_name in self.bayes_settings.diagnostics + for diag_name in self.blackbox_settings.diagnostics } result["MODEL_DATA"]["SAMPLE_IDX"] = np.arange(self.optimiser_context.optimiser_settings.iterations* (1-self.optimiser_context.optimiser_settings.burn_frac)) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index e3a92af8..f808237e 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -265,34 +265,30 @@ def dict_of_dataarray_to_numpy(dict_of_dataarray): return dict_of_dataarray -@dataclass -class BayesSettings: - diagnostics: list = field(default_factory=lambda: DEFAULT_DIAG_NAMES) - param_names: list = field(default_factory=lambda: OPTIMISED_PARAMS) - opt_quantity: list = field(default_factory=lambda: OPTIMISED_QUANTITY) - priors: dict = field(default_factory=lambda: DEFAULT_PRIORS) - percent_error: float = 0.10 - +def sample_from_priors(param_names: list, priors: dict, size=10): """ - TODO: default methods / getter + setters - print warning if using default values + TODO: may be able to remove param_names from here and at some point earlier remove priors that aren't used + then loop over remaining priors while handling conditional priors somehow... + The order of samples may need to be checked / reordered at some point then """ - def __post_init__(self): - missing_quantities = [quant for quant in self.opt_quantity if quant.split(".")[0] not in self.diagnostics] - if missing_quantities: - raise ValueError(f"{missing_quantities} missing the relevant diagnostic") - - # check all priors are defined - for name in self.param_names: - if name in self.priors.keys(): - if hasattr(self.priors[name], "rvs"): - continue - else: - raise TypeError(f"prior object {name} missing rvs method") - else: - raise ValueError(f"Missing prior for {name}") - + # Throw out samples that don't meet conditional priors and redraw + samples = np.empty((param_names.__len__(), 0)) + while samples.size < param_names.__len__() * size: + # Some mangling of dictionaries so _ln_prior works + # Increase size * n if too slow / looping too much + new_sample = { + name: priors[name].rvs(size=size * 2) for name in param_names + } + _ln_prior = ln_prior(priors, new_sample) + # Convert from dictionary of arrays -> array, + # then filtering out where ln_prior is -infinity + accepted_samples = np.array(list(new_sample.values()))[ + :, _ln_prior != -np.inf + ] + samples = np.append(samples, accepted_samples, axis=1) + samples = samples[:, 0:size] + return samples.transpose() @dataclass class PlasmaSettings: @@ -731,31 +727,6 @@ def run(self): results = None return results -def sample_from_priors(param_names: list, priors: dict, size=10): - """ - TODO: may be able to remove param_names from here and at some point earlier remove priors that aren't used - then loop over remaining priors while handling conditional priors somehow... - The order of samples may need to be checked / reordered at some point then - """ - - # Throw out samples that don't meet conditional priors and redraw - samples = np.empty((param_names.__len__(), 0)) - while samples.size < param_names.__len__() * size: - # Some mangling of dictionaries so _ln_prior works - # Increase size * n if too slow / looping too much - new_sample = { - name: priors[name].rvs(size=size * 2) for name in param_names - } - _ln_prior = ln_prior(priors, new_sample) - # Convert from dictionary of arrays -> array, - # then filtering out where ln_prior is -infinity - accepted_samples = np.array(list(new_sample.values()))[ - :, _ln_prior != -np.inf - ] - samples = np.append(samples, accepted_samples, axis=1) - samples = samples[:, 0:size] - return samples.transpose() - @dataclass class EmceeOptimiser(OptimiserContext): @@ -876,10 +847,40 @@ def format_results(self): return results + +@dataclass +class BayesBBSettings: + diagnostics: list = field(default_factory=lambda: DEFAULT_DIAG_NAMES) + param_names: list = field(default_factory=lambda: OPTIMISED_PARAMS) + opt_quantity: list = field(default_factory=lambda: OPTIMISED_QUANTITY) + priors: dict = field(default_factory=lambda: DEFAULT_PRIORS) + percent_error: float = 0.10 + + """ + TODO: default methods / getter + setters + print warning if using default values + """ + + def __post_init__(self): + missing_quantities = [quant for quant in self.opt_quantity if quant.split(".")[0] not in self.diagnostics] + if missing_quantities: + raise ValueError(f"{missing_quantities} missing the relevant diagnostic") + + # check all priors are defined + for name in self.param_names: + if name in self.priors.keys(): + if hasattr(self.priors[name], "rvs"): + continue + else: + raise TypeError(f"prior object {name} missing rvs method") + else: + raise ValueError(f"Missing prior for {name}") + + class BayesWorkflow(AbstractBayesWorkflow): def __init__( self, - bayes_settings: BayesSettings, + blackbox_settings: BayesBBSettings, data_context: DataContext, plasma_context: PlasmaContext, model_context: ModelContext, @@ -890,7 +891,7 @@ def __init__( dt: float = 0.01, ): - self.bayes_settings = bayes_settings + self.blackbox_settings = blackbox_settings self.data_context = data_context self.plasma_context = plasma_context self.model_context = model_context @@ -913,9 +914,9 @@ def __init__( self.blackbox = BayesBlackBox(data=self.data_context.opt_data, plasma_context=self.plasma_context, model_context=self.model_context, - quant_to_optimise=self.bayes_settings.opt_quantity, - priors=self.bayes_settings.priors, - percent_error=self.bayes_settings.percent_error) + quant_to_optimise=self.blackbox_settings.opt_quantity, + priors=self.blackbox_settings.priors, + percent_error=self.blackbox_settings.percent_error) self.optimiser_context.init_optimiser(self.blackbox.ln_posterior, model_kwargs=self.model_call_kwargs) From 23150ebbe9a4a5df1099877cb0a62d95f0513c17 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 13:17:30 +0100 Subject: [PATCH 223/288] renamed bayes_settings -> blackbox_settings --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index f808237e..79a6f084 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -971,7 +971,7 @@ def __call__( self.node_structure = create_nodes( pulse_to_write=pulse_to_write, run=run, - diagnostic_quantities=self.bayes_settings.opt_quantity, + diagnostic_quantities=self.blackbox_settings.opt_quantity, mode="NEW", ) write_nodes(pulse_to_write, self.node_structure, self.result) From d1a7654e7bf2422219c818bae8a694bbfc80d031 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 13:17:47 +0100 Subject: [PATCH 224/288] renamed bayes_settings -> blackbox_settings --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 79a6f084..66f6be19 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -1010,7 +1010,7 @@ def __call__( ] # BlackBoxSettings - bayes_settings = BayesSettings(diagnostics=diagnostics, param_names=opt_params, + bayes_settings = BayesBBSettings(diagnostics=diagnostics, param_names=opt_params, opt_quantity=opt_quant, priors=DEFAULT_PRIORS, ) data_settings = ReaderSettings(filters={}, From 59071acd7df104c5d50a11140631e595b760ab3c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 13:18:08 +0100 Subject: [PATCH 225/288] renamed bayes_settings -> blackbox_settings --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 66f6be19..497c3d04 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -1057,7 +1057,7 @@ def __call__( optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) workflow = BayesWorkflow(tstart=tstart, tend=tend, dt=dt, - bayes_settings=bayes_settings, data_context=data_context, + blackbox_settings=bayes_settings, data_context=data_context, optimiser_context=optimiser_context, plasma_context=plasma_context, model_context=model_context) From 8e3cfa177b7d3b006f7d348c4f14dfc866a12a90 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 13:19:51 +0100 Subject: [PATCH 226/288] added mock transforms to default_factory --- indica/workflows/bayes_workflow.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 497c3d04..f060afd9 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -674,7 +674,10 @@ def process_data(self, model_callable: Callable): @dataclass class MockData(PhantomData): - diagnostic_transforms: dict = field(default_factory=lambda: {}) + diagnostic_transforms: dict = field(default_factory=lambda: {"xrcs": helike_transform_example(1), + "smmh1": smmh1_transform_example(1), + "cxff_pi": pi_transform_example(5), + "ts": ts_transform_example(11), }) def read_data(self): print("Reading mock equilibrium / transforms") From dc3d7a3a2341b02acc13c60d75e39d6f66d66f71 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 13:20:44 +0100 Subject: [PATCH 227/288] cxfi_tws_c example transform added --- indica/workflows/bayes_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index f060afd9..caa90f5b 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -677,6 +677,7 @@ class MockData(PhantomData): diagnostic_transforms: dict = field(default_factory=lambda: {"xrcs": helike_transform_example(1), "smmh1": smmh1_transform_example(1), "cxff_pi": pi_transform_example(5), + "cxff_tws_c": pi_transform_example(3), "ts": ts_transform_example(11), }) def read_data(self): From 1f4108fe38e1fa07471320df5e9fd1b3d686e751 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 14:19:22 +0100 Subject: [PATCH 228/288] ModelSettings dataclass created to handle model initialisation and calling settings --- indica/workflows/abstract_bayes_workflow.py | 2 +- indica/workflows/bayes_workflow.py | 144 ++++++++++---------- 2 files changed, 71 insertions(+), 75 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index d2e1be2a..b98b76f3 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -49,7 +49,7 @@ def _build_inputs_dict(self): "BURN_FRAC": self.optimiser_context.optimiser_settings.burn_frac, "ITER": self.optimiser_context.optimiser_settings.iterations, "NWALKERS": self.optimiser_context.optimiser_settings.nwalkers, - "MODEL_KWARGS": self.model_call_kwargs, + "MODEL_KWARGS": self.model_context.model_settings.init_kwargs, "OPT_QUANTITY": self.blackbox_settings.opt_quantity, "PARAM_NAMES": self.blackbox_settings.param_names, "PULSE": self.data_context.pulse, diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index caa90f5b..b60a5503 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -93,7 +93,6 @@ "Ti_prof.wped": get_uniform(1, 6), "Ti_prof.wcenter": get_uniform(0.2, 0.4), "Ti_prof.peaking": get_uniform(1, 6), - "xrcs.pixel_offset": get_uniform(-4.01, -4.0), } OPTIMISED_PARAMS = [ @@ -213,6 +212,7 @@ def sample_with_moments(sampler, start_points, iterations, n_params, auto_sample ] return autocorr + def sample_with_autocorr(sampler, start_points, iterations, n_params, auto_sample=5, ): autocorr = np.ones(shape=(iterations, n_params)) * np.nan old_tau = np.inf @@ -290,6 +290,7 @@ def sample_from_priors(param_names: list, priors: dict, size=10): samples = samples[:, 0:size] return samples.transpose() + @dataclass class PlasmaSettings: main_ion: str = "h" @@ -444,13 +445,26 @@ def save_phantom_profiles(self, kinetic_profiles=None, phantoms=None): # self.plasma.Nimp_prof = lambda: Nimp.values +@dataclass +class ModelSettings: + init_kwargs: dict = field(default_factory=lambda: { + "cxff_pi": {"element": "ar"}, + "cxff_tws_c": {"element": "c"}, + "xrcs": { + "window_masks": [slice(0.394, 0.396)], + }, + }) + call_kwargs: dict = field(default_factory=lambda: { + "xrcs": {"pixel_offset": -4.0} + }) + @dataclass class ModelContext: + model_settings: ModelSettings diagnostics: list plasma_context: PlasmaContext equilibrium: Equilibrium transforms: dict - model_kwargs: Dict[str, Dict[str, Any]] """ Setup models so that they have transforms / plasma / equilibrium etc.. everything needed to produce bckc from models @@ -458,15 +472,18 @@ class ModelContext: TODO: remove repeating code / likely make general methods """ - def update_model_kwargs(self, data: dict): - if self.model_kwargs is None: - self.model_kwargs = {} + def __post_init__(self): + # Create empty dictionaries for diagnostics where no init or call kwargs defined for diag in self.diagnostics: - if diag not in self.model_kwargs.keys(): - self.model_kwargs[diag] = {} + if diag not in self.model_settings.init_kwargs.keys(): + self.model_settings.init_kwargs[diag] = {} + if diag not in self.model_settings.call_kwargs.keys(): + self.model_settings.call_kwargs[diag] = {} + + def update_model_kwargs(self, data: dict): if "xrcs" in data.keys(): - self.model_kwargs["xrcs"]["window"] = data["xrcs"]["spectra"].wavelength.values + self.model_settings.init_kwargs["xrcs"]["window"] = data["xrcs"]["spectra"].wavelength.values # TODO: handling model calls dependent on exp data background = data["xrcs"]["spectra"].where( @@ -474,7 +491,7 @@ def update_model_kwargs(self, data: dict): & (data["xrcs"]["spectra"].wavelength > 0.388), drop=True, ) - self.model_kwargs["xrcs"]["background"] = background.mean(dim="wavelength") + self.model_settings.init_kwargs["xrcs"]["background"] = background.mean(dim="wavelength") def init_models(self, ): if not hasattr(self, "plasma_context"): @@ -486,30 +503,30 @@ def init_models(self, ): for diag in self.diagnostics: if diag == "smmh1": self.transforms[diag].set_equilibrium(self.equilibrium, force=True) - self.models[diag] = Interferometry(name=diag, **self.model_kwargs[diag]) + self.models[diag] = Interferometry(name=diag, **self.model_settings.init_kwargs[diag]) self.models[diag].set_los_transform(self.transforms[diag]) elif diag == "efit": - self.models[diag] = EquilibriumReconstruction(name=diag, **self.model_kwargs[diag]) + self.models[diag] = EquilibriumReconstruction(name=diag, **self.model_settings.init_kwargs[diag]) elif diag == "cxff_pi": self.transforms[diag].set_equilibrium(self.equilibrium, force=True) - self.models[diag] = ChargeExchange(name=diag, **self.model_kwargs[diag]) + self.models[diag] = ChargeExchange(name=diag, **self.model_settings.init_kwargs[diag]) self.models[diag].set_transect_transform(self.transforms[diag]) elif diag == "cxff_tws_c": self.transforms[diag].set_equilibrium(self.equilibrium, force=True) - self.models[diag] = ChargeExchange(name=diag, **self.model_kwargs[diag]) + self.models[diag] = ChargeExchange(name=diag, **self.model_settings.init_kwargs[diag]) self.models[diag].set_transect_transform(self.transforms[diag]) elif diag == "ts": self.transforms[diag].set_equilibrium(self.equilibrium, force=True) - self.models[diag] = ThomsonScattering(name=diag, **self.model_kwargs[diag]) + self.models[diag] = ThomsonScattering(name=diag, **self.model_settings.init_kwargs[diag]) self.models[diag].set_transect_transform(self.transforms[diag]) elif diag == "xrcs": self.transforms[diag].set_equilibrium(self.equilibrium, force=True) - self.models[diag] = HelikeSpectrometer(name="xrcs", **self.model_kwargs[diag]) + self.models[diag] = HelikeSpectrometer(name="xrcs", **self.model_settings.init_kwargs[diag]) self.models[diag].set_los_transform(self.transforms[diag]) else: raise ValueError(f"{diag} not implemented in ModelHandler.setup_models") @@ -519,45 +536,41 @@ def init_models(self, ): return self.models - def _build_bckc(self, params: dict = None, **kwargs): + def _build_bckc(self, params: dict = None, ): """ Parameters ---------- params - dictionary which is updated by optimiser - kwargs - passed to model call i.e. settings Returns ------- nested bckc of results """ - # Float128 since rounding of small numbers causes problems - # when initial results are bad fits - # model_data = self.bckc[key].astype("float128") if params is None: params = {} self.bckc: dict = {} for model_name, model in self.models.items(): # removes "model.name." from params and kwargs then passes them to model # e.g. xrcs.background -> background - _nuisance_params = { + _call_params = { param_name.replace(model.name + ".", ""): param_value for param_name, param_value in params.items() if model.name in param_name } - _model_settings = { - kwarg_name.replace(model.name + ".", ""): kwarg_value - for kwarg_name, kwarg_value in kwargs.items() - if model.name in kwarg_name + # call_kwargs defined in model_settings + _call_kwargs = { + kwarg_name: kwarg_value + for kwarg_name, kwarg_value in self.model_settings.call_kwargs[model_name].items() } - _model_kwargs = { - **_nuisance_params, - **_model_settings, + **_call_kwargs, + **_call_params, } # combine dictionaries + _bckc = model(**_model_kwargs) _model_bckc = {model.name: {value_name: value for value_name, value in _bckc.items()}} - # prepend model name to bckc + # prepend model name to bckc self.bckc = dict(self.bckc, **_model_bckc) return self.bckc @@ -649,6 +662,7 @@ def process_data(self, model_callable: Callable): opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 self.opt_data = opt_data + @dataclass class PhantomData(DataContext): phantoms = True @@ -672,6 +686,7 @@ def process_data(self, model_callable: Callable): self.pre_process_data(model_callable) self.opt_data = flatdict.FlatDict(self.binned_data, ".") + @dataclass class MockData(PhantomData): diagnostic_transforms: dict = field(default_factory=lambda: {"xrcs": helike_transform_example(1), @@ -735,8 +750,7 @@ def run(self): @dataclass class EmceeOptimiser(OptimiserContext): - def init_optimiser(self, blackbox_func: Callable, model_kwargs: dict = field(default={}) - ): + def init_optimiser(self, blackbox_func: Callable): ndim = len(self.optimiser_settings.param_names) self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] self.optimiser = emcee.EnsembleSampler( @@ -745,7 +759,6 @@ def init_optimiser(self, blackbox_func: Callable, model_kwargs: dict = field(def log_prob_fn=blackbox_func, parameter_names=self.optimiser_settings.param_names, moves=self.move, - kwargs=model_kwargs, ) def sample_start_points(self, ): @@ -770,7 +783,6 @@ def sample_start_points(self, ): raise ValueError(f"Sample method: {self.optimiser_settings.sample_method} " f"not recognised, Defaulting to random sampling") - def sample_from_high_density_region( self, param_names: list, priors: dict, optimiser, nwalkers: int, nsamples=100 ): @@ -811,7 +823,7 @@ def run(self, ): self.optimiser_settings.param_names.__len__(), auto_sample=10, stopping_factor=self.optimiser_settings.stopping_criterion_factor, - debug = self.optimiser_settings.stopping_criteria_debug + debug=self.optimiser_settings.stopping_criteria_debug ) else: raise ValueError(f"Stopping criterion: {self.optimiser_settings.stopping_criterion} not recognised") @@ -844,14 +856,13 @@ def format_results(self): discard=int(self.optimiser.iteration * self.optimiser_settings.burn_frac), flat=True) # pad index dim with maximum number of iterations max_iter = self.optimiser_settings.iterations * self.optimiser_settings.nwalkers - npad = ((0, int(max_iter*(1-self.optimiser_settings.burn_frac) - post_sample.shape[0])), (0, 0)) + npad = ((0, int(max_iter * (1 - self.optimiser_settings.burn_frac) - post_sample.shape[0])), (0, 0)) results["post_sample"] = np.pad(post_sample, npad, constant_values=np.nan) results["auto_corr"] = self.autocorr return results - @dataclass class BayesBBSettings: diagnostics: list = field(default_factory=lambda: DEFAULT_DIAG_NAMES) @@ -912,8 +923,7 @@ def __init__( self.model_context.update_model_kwargs(self.data_context.binned_data) self.model_context.init_models() - self.model_call_kwargs = {"xrcs.pixel_offset": -4.0} - self.data_context.process_data(partial(self.model_context._build_bckc, **self.model_call_kwargs)) + self.data_context.process_data(self.model_context._build_bckc, ) self.blackbox = BayesBlackBox(data=self.data_context.opt_data, plasma_context=self.plasma_context, @@ -922,7 +932,7 @@ def __init__( priors=self.blackbox_settings.priors, percent_error=self.blackbox_settings.percent_error) - self.optimiser_context.init_optimiser(self.blackbox.ln_posterior, model_kwargs=self.model_call_kwargs) + self.optimiser_context.init_optimiser(self.blackbox.ln_posterior, ) def __call__( self, @@ -999,59 +1009,45 @@ def __call__( ] # diagnostic_quantities opt_quant = [ - "xrcs.spectra", - # "efit.wp", - "ts.te" - ] + "xrcs.spectra", + # "efit.wp", + "ts.te" + ] opt_params = [ - "Te_prof.y0", - "Te_prof.peaking", - "Te_prof.wped", - "Te_prof.wcenter", + "Te_prof.y0", + "Te_prof.peaking", + "Te_prof.wped", + "Te_prof.wcenter", - # "Ti_prof.y0", - # "Ne_prof.y0", - ] + # "Ti_prof.y0", + # "Ne_prof.y0", + ] # BlackBoxSettings bayes_settings = BayesBBSettings(diagnostics=diagnostics, param_names=opt_params, - opt_quantity=opt_quant, priors=DEFAULT_PRIORS, ) + opt_quantity=opt_quant, priors=DEFAULT_PRIORS, ) data_settings = ReaderSettings(filters={}, revisions={}) # Add general methods for filtering data co-ords to ReadST40 - - # mock_transforms = {"xrcs": helike_transform_example(1), - # "smmh1": smmh1_transform_example(1), - # "cxff_pi": pi_transform_example(5), - # "ts": ts_transform_example(11), } - # data_context = MockData(pulse=None, diagnostic_transforms=mock_transforms, ) - # - # data_context = ExpData(pulse=pulse, diagnostics=diagnostics, - # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) - # data_context.read_data() - + # data_context = MockData(pulse=pulse, diagnostics=diagnostics, + # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + # data_context = PhantomData(pulse=pulse, diagnostics=diagnostics, + # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context = ExpData(pulse=pulse, diagnostics=diagnostics, tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context.read_data() - plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), n_rad=20) plasma_context = PlasmaContext(plasma_settings=plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) - model_init_kwargs = { - "cxff_pi": {"element": "ar"}, - "cxff_tws_c": {"element": "c"}, - "xrcs": { - "window_masks": [slice(0.394, 0.396)], - }, - } + model_settings = ModelSettings(call_kwargs={"xrcs": {"pixel_offset": -4.0}}) model_context = ModelContext(diagnostics=diagnostics, plasma_context=plasma_context, equilibrium=data_context.equilibrium, transforms=data_context.transforms, - model_kwargs=model_init_kwargs, + model_settings=model_settings, ) optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=5, @@ -1061,8 +1057,8 @@ def __call__( optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) workflow = BayesWorkflow(tstart=tstart, tend=tend, dt=dt, - blackbox_settings=bayes_settings, data_context=data_context, - optimiser_context=optimiser_context, - plasma_context=plasma_context, model_context=model_context) + blackbox_settings=bayes_settings, data_context=data_context, + optimiser_context=optimiser_context, + plasma_context=plasma_context, model_context=model_context) workflow(pulse_to_write=25000000, run="RUN01", mds_write=True, plot=True, filepath="./results/test/") From c9cb3447fd9144e1218cd0ecf6a507de411c1703 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 14:25:25 +0100 Subject: [PATCH 229/288] missing .self --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index b60a5503..51b6356f 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -670,7 +670,7 @@ class PhantomData(DataContext): def read_data(self, ): self.reader = ReadST40(self.pulse, tstart=self.tstart, tend=self.tend, dt=self.dt, ) self.reader(self.diagnostics, revisions=self.reader_settings.revisions, filters=self.reader_settings.filters) - missing_keys = set(diagnostics) - set(self.reader.binned_data.keys()) + missing_keys = set(self.diagnostics) - set(self.reader.binned_data.keys()) if len(missing_keys) > 0: raise ValueError(f"missing data: {missing_keys}") self.equilibrium = self.reader.equilibrium From 8fbc4ee290312caa46747f5a13e4806a8b1402dd Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 14:40:11 +0100 Subject: [PATCH 230/288] stopping criteria sampling rate added to OptimiserSettings --- indica/workflows/bayes_workflow.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 51b6356f..b2e92b45 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -720,8 +720,9 @@ class OptimiserEmceeSettings: burn_frac: float = 0.20 sample_method: str = "random" starting_samples: int = 100 - stopping_criterion: str = "mode" - stopping_criterion_factor: float = 0.01 + stopping_criteria: str = "mode" + stopping_criteria_factor: float = 0.01 + stopping_criteria_sample: int = 20, stopping_criteria_debug: bool = False @@ -815,18 +816,18 @@ def sample_from_high_density_region( def run(self, ): - if self.optimiser_settings.stopping_criterion == "mode": + if self.optimiser_settings.stopping_criteria == "mode": self.autocorr = sample_with_moments( self.optimiser, self.start_points, self.optimiser_settings.iterations, self.optimiser_settings.param_names.__len__(), - auto_sample=10, - stopping_factor=self.optimiser_settings.stopping_criterion_factor, + auto_sample=self.optimiser_settings.stopping_criteria_sample, + stopping_factor=self.optimiser_settings.stopping_criteria_factor, debug=self.optimiser_settings.stopping_criteria_debug ) else: - raise ValueError(f"Stopping criterion: {self.optimiser_settings.stopping_criterion} not recognised") + raise ValueError(f"Stopping criteria: {self.optimiser_settings.stopping_criteria} not recognised") optimiser_results = self.format_results() return optimiser_results @@ -1052,7 +1053,7 @@ def __call__( optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=5, sample_method="high_density", starting_samples=100, burn_frac=0.20, - stopping_criterion="mode", stopping_criterion_factor=0.01, + stopping_criteria="mode", stopping_criteria_factor=0.01, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) From b0f62e4d36b7227ed44e2f6f9e483d566cd1e0e9 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 24 Oct 2023 15:00:16 +0100 Subject: [PATCH 231/288] typo --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index b2e92b45..21c192d8 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -722,7 +722,7 @@ class OptimiserEmceeSettings: starting_samples: int = 100 stopping_criteria: str = "mode" stopping_criteria_factor: float = 0.01 - stopping_criteria_sample: int = 20, + stopping_criteria_sample: int = 20 stopping_criteria_debug: bool = False From 5f791b7b42729d221c6870b08625e7f986f020c1 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 25 Oct 2023 09:30:58 +0100 Subject: [PATCH 232/288] added moments tests --- indica/workflows/bda_run.py | 188 +++++++++++++++++++++++++++--------- 1 file changed, 142 insertions(+), 46 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index 3cea8645..addd8a34 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -1,5 +1,7 @@ -from indica.workflows.bayes_workflow import BayesWorkflow, DEFAULT_PRIORS, DEFAULT_PROFILE_PARAMS, BayesSettings, ReaderSettings, \ - ExpData, MockData, PhantomData, PlasmaSettings, PlasmaContext, ModelContext, EmceeOptimiser, OptimiserEmceeSettings +from indica.workflows.bayes_workflow import BayesWorkflow, DEFAULT_PRIORS, DEFAULT_PROFILE_PARAMS, BayesBBSettings, \ + ReaderSettings, \ + ExpData, PhantomData, PlasmaSettings, PlasmaContext, ModelContext, EmceeOptimiser, OptimiserEmceeSettings, \ + ModelSettings PARAMS_DEFAULT =[ "Ne_prof.y1", @@ -13,14 +15,13 @@ # "Nimp_prof.wped", "Nimp_prof.peaking", "Te_prof.y0", - # "Te_prof.wped", - # "Te_prof.wcenter", + "Te_prof.wped", + "Te_prof.wcenter", "Te_prof.peaking", "Ti_prof.y0", # "Ti_prof.wped", "Ti_prof.wcenter", "Ti_prof.peaking", - "xrcs.pixel_offset", ] DIAG_DEFAULT = [ @@ -35,7 +36,7 @@ DIAG_DEFAULT_CHERS = [ "xrcs", "ts", - "efit", + # "efit", # "cxff_pi", "cxff_tws_c" # "smmh1", @@ -44,7 +45,7 @@ DIAG_DEFAULT_PI = [ "xrcs", "ts", - "efit", + # "efit", "cxff_pi", # "cxff_tws_c" # "smmh1", @@ -54,94 +55,189 @@ "xrcs.spectra", "ts.ne", "ts.te", - "efit.wp", + # "efit.wp", "cxff_pi.ti", "cxff_tws_c.ti", # "smmh1.ne" ] -def bda_run(pulse, diagnostics, param_names, opt_quantity, +def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, tstart=0.02, tend=0.03, dt=0.01, revisions = {}, filters={}, - iterations=500, nwalkers=50, mds_write=False, plot=False, - run="RUN01", **kwargs): + iterations=500, nwalkers=50, stopping_criteria_factor=0.01, + mds_write=False, plot=False, run="RUN01", dirname=None, **kwargs): + + if dirname is None: + dirname = f"{pulse}" # BlackBoxSettings - bayes_settings = BayesSettings(diagnostics=diagnostics, param_names=param_names, - opt_quantity=opt_quantity, priors=DEFAULT_PRIORS, ) + bayes_settings = BayesBBSettings(diagnostics=diagnostics, param_names=param_names, + opt_quantity=opt_quantity, priors=DEFAULT_PRIORS, ) data_settings = ReaderSettings(filters=filters, revisions=revisions) - - data_context = ExpData(pulse=pulse, diagnostics=diagnostics, - tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + if phantom: + data_context = PhantomData(pulse=pulse, diagnostics=diagnostics, + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + else: + data_context = ExpData(pulse=pulse, diagnostics=diagnostics, + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context.read_data() plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), n_rad=20) plasma_context = PlasmaContext(plasma_settings=plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) - model_init_kwargs = { - "cxff_pi": {"element": "ar"}, - "cxff_tws_c": {"element": "c"}, - "xrcs": { - "window_masks": [slice(0.394, 0.396)], - }, - } + model_settings = ModelSettings() model_context = ModelContext(diagnostics=diagnostics, plasma_context=plasma_context, equilibrium=data_context.equilibrium, transforms=data_context.transforms, - model_kwargs=model_init_kwargs, + model_settings=model_settings, ) optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=nwalkers, iterations=iterations, sample_method="high_density", starting_samples=100, burn_frac=0.20, - stopping_criterion="mode", stopping_criterion_factor=10, - priors=bayes_settings.priors) + stopping_criteria="mode", stopping_criteria_factor=stopping_criteria_factor, + stopping_criteria_debug=True, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) workflow = BayesWorkflow(tstart=tstart, tend=tend, dt=dt, - bayes_settings=bayes_settings, data_context=data_context, + blackbox_settings=bayes_settings, data_context=data_context, optimiser_context=optimiser_context, plasma_context=plasma_context, model_context=model_context) - workflow(pulse_to_write=25000000+pulse, run=run, mds_write=mds_write, plot=plot, filepath=f"./results/{pulse}/") + workflow(pulse_to_write=25000000+pulse, run=run, mds_write=mds_write, plot=plot, filepath=f"./results/{dirname}/") if __name__ == "__main__": pulse_info = [ + # stopping criteria test / integration test + + [(11226, + ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + PARAMS_DEFAULT, + [ + "xrcs.spectra", + "cxff_pi.ti", + "cxff_tws_c.ti", + "ts.ne", + "ts.te", + ]), + dict( + phantom=True, + tstart=0.07, + tend=0.08, + dt=0.01, + iterations=5000, + nwalkers=50, + stopping_criteria_factor=0.02, + mds_write=True, + plot=True, + run="RUN01", + dirname=f"{11226}_phantom_moment_02", + )], + + [(11226, + ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + PARAMS_DEFAULT, + [ + "xrcs.spectra", + "cxff_pi.ti", + "cxff_tws_c.ti", + "ts.ne", + "ts.te", + ]), + dict( + phantom=True, + tstart=0.07, + tend=0.08, + dt=0.01, + iterations=5000, + nwalkers=50, + stopping_criteria_factor=0.01, + mds_write=True, + plot=True, + run="RUN01", + dirname=f"{11226}_phantom_moment_01", + )], - [(11366, - ["ts"], - ["Te_prof.y0", - "Te_prof.peaking", - "Te_prof.wped", - "Te_prof.wcenter", - ], - [ - # "xrcs.spectra", - # "ts.ne", - "ts.te", - # "efit.wp", - # "cxff_pi.ti", - # "cxff_tws_c.ti", - # "smmh1.ne" - ]), + [(11226, + ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + PARAMS_DEFAULT, + [ + "xrcs.spectra", + "cxff_pi.ti", + "cxff_tws_c.ti", + "ts.ne", + "ts.te", + ]), dict( + phantom=True, tstart=0.07, tend=0.08, dt=0.01, - iterations=500, + iterations=5000, nwalkers=50, + stopping_criteria_factor=0.001, mds_write=True, plot=True, run="RUN01", - )], + dirname=f"{11226}_phantom_moment_001", + )], + + [(11226, + ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + PARAMS_DEFAULT, + [ + "xrcs.spectra", + "cxff_pi.ti", + "cxff_tws_c.ti", + "ts.ne", + "ts.te", + ]), + dict( + phantom=False, + tstart=0.07, + tend=0.08, + dt=0.01, + iterations=10, + nwalkers=50, + mds_write=True, + plot=True, + run="RUN01", + dirname=f"{11226}_exp", + )], + + + + # [(10009, + # ["xrcs", "cxff_pi"], + # PARAMS_DEFAULT, + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # ]), + # dict( + # phantom=True, + # tstart=0.07, + # tend=0.08, + # dt=0.01, + # iterations=100, + # nwalkers=50, + # mds_write=True, + # plot=True, + # run="RUN01", + # dirname=f"{11336}_phantom", + # )], ] + + + for info in pulse_info: + print(f"pulse: {info[0][0]}") bda_run(*info[0], **info[1]) \ No newline at end of file From 5cc2c2de5f4d348b9b2aa7988bad90fd7640b09c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 25 Oct 2023 09:31:26 +0100 Subject: [PATCH 233/288] adjusted nimp.y0 prior --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 21c192d8..50adb3f8 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -71,7 +71,7 @@ "Ne_prof.wped": get_uniform(2, 6), "Ne_prof.wcenter": get_uniform(0.2, 0.4), "Ne_prof.peaking": get_uniform(1, 4), - "Nimp_prof.y0": loguniform(1e15, 1e18), + "Nimp_prof.y0": loguniform(1e16, 1e18), "Nimp_prof.y1": loguniform(1e14, 1e16), "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 From ffd70db2c837ee528fcf8c462fc2bac3951f6531 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 25 Oct 2023 10:04:01 +0100 Subject: [PATCH 234/288] example run updated --- indica/workflows/bayes_workflow.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 50adb3f8..e839933b 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -1001,16 +1001,16 @@ def __call__( dt = 0.01 diagnostics = [ - "xrcs", - "efit", + # "xrcs", + # "efit", # "smmh1", # "cxff_pi", - "cxff_tws_c", + # "cxff_tws_c", "ts", ] # diagnostic_quantities opt_quant = [ - "xrcs.spectra", + # "xrcs.spectra", # "efit.wp", "ts.te" ] @@ -1051,7 +1051,7 @@ def __call__( model_settings=model_settings, ) - optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=5, + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=100, sample_method="high_density", starting_samples=100, burn_frac=0.20, stopping_criteria="mode", stopping_criteria_factor=0.01, priors=bayes_settings.priors) From 54fb9683a0e6d41aeb7876f3f66b2d81aed7d76e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 26 Oct 2023 09:33:22 +0100 Subject: [PATCH 235/288] more tests for walkers --- indica/workflows/bda_run.py | 94 +++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 9 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index addd8a34..ea87be1d 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -115,7 +115,6 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, pulse_info = [ # stopping criteria test / integration test - [(11226, ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], PARAMS_DEFAULT, @@ -132,12 +131,12 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, tend=0.08, dt=0.01, iterations=5000, - nwalkers=50, - stopping_criteria_factor=0.02, + nwalkers=26, + stopping_criteria_factor=0.01, mds_write=True, plot=True, run="RUN01", - dirname=f"{11226}_phantom_moment_02", + dirname=f"{11226}_phantom_walkers_26", )], [(11226, @@ -161,7 +160,7 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, mds_write=True, plot=True, run="RUN01", - dirname=f"{11226}_phantom_moment_01", + dirname=f"{11226}_phantom_walkers_50", )], [(11226, @@ -180,14 +179,87 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, tend=0.08, dt=0.01, iterations=5000, - nwalkers=50, - stopping_criteria_factor=0.001, + nwalkers=100, + stopping_criteria_factor=0.01, mds_write=True, plot=True, run="RUN01", - dirname=f"{11226}_phantom_moment_001", + dirname=f"{11226}_phantom_walkers_100", )], + + # [(11226, + # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + # PARAMS_DEFAULT, + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "cxff_tws_c.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=True, + # tstart=0.07, + # tend=0.08, + # dt=0.01, + # iterations=5000, + # nwalkers=50, + # stopping_criteria_factor=0.02, + # mds_write=True, + # plot=True, + # run="RUN01", + # dirname=f"{11226}_phantom_moment_02", + # )], + # + # [(11226, + # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + # PARAMS_DEFAULT, + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "cxff_tws_c.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=True, + # tstart=0.07, + # tend=0.08, + # dt=0.01, + # iterations=5000, + # nwalkers=50, + # stopping_criteria_factor=0.01, + # mds_write=True, + # plot=True, + # run="RUN01", + # dirname=f"{11226}_phantom_moment_01", + # )], + # + # [(11226, + # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + # PARAMS_DEFAULT, + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "cxff_tws_c.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=True, + # tstart=0.07, + # tend=0.08, + # dt=0.01, + # iterations=5000, + # nwalkers=50, + # stopping_criteria_factor=0.001, + # mds_write=True, + # plot=True, + # run="RUN01", + # dirname=f"{11226}_phantom_moment_001", + # )], + [(11226, ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], PARAMS_DEFAULT, @@ -203,7 +275,8 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, tstart=0.07, tend=0.08, dt=0.01, - iterations=10, + iterations=5000, + stopping_criteria_factor=0.01, nwalkers=50, mds_write=True, plot=True, @@ -233,6 +306,9 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # dirname=f"{11336}_phantom", # )], + + # RFX low ti/te pulse + ] From 7e72f8d6f599e17ba05062e465eadeccc358d99f Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 26 Oct 2023 10:05:56 +0100 Subject: [PATCH 236/288] typo --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index e839933b..2a38a270 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -615,7 +615,7 @@ def pre_process_data(self, model_callable: Callable): @abstractmethod def process_data(self, model_callable: Callable): - self.pre_process_data(model_context) + self.pre_process_data(model_callable) self.opt_data = flatdict.FlatDict(self.binned_data) From d54840ddd0e8e91f88e7316854c9d7a724e98010 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 30 Oct 2023 16:08:02 +0000 Subject: [PATCH 237/288] 11089 pulse added --- indica/workflows/bda_run.py | 212 ++++++++++++++++++++---------------- 1 file changed, 116 insertions(+), 96 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index ea87be1d..93323d61 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -115,77 +115,77 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, pulse_info = [ # stopping criteria test / integration test - [(11226, - ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - PARAMS_DEFAULT, - [ - "xrcs.spectra", - "cxff_pi.ti", - "cxff_tws_c.ti", - "ts.ne", - "ts.te", - ]), - dict( - phantom=True, - tstart=0.07, - tend=0.08, - dt=0.01, - iterations=5000, - nwalkers=26, - stopping_criteria_factor=0.01, - mds_write=True, - plot=True, - run="RUN01", - dirname=f"{11226}_phantom_walkers_26", - )], - - [(11226, - ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - PARAMS_DEFAULT, - [ - "xrcs.spectra", - "cxff_pi.ti", - "cxff_tws_c.ti", - "ts.ne", - "ts.te", - ]), - dict( - phantom=True, - tstart=0.07, - tend=0.08, - dt=0.01, - iterations=5000, - nwalkers=50, - stopping_criteria_factor=0.01, - mds_write=True, - plot=True, - run="RUN01", - dirname=f"{11226}_phantom_walkers_50", - )], + # [(11226, + # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + # PARAMS_DEFAULT, + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "cxff_tws_c.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=True, + # tstart=0.07, + # tend=0.08, + # dt=0.01, + # iterations=5000, + # nwalkers=26, + # stopping_criteria_factor=0.01, + # mds_write=True, + # plot=True, + # run="RUN01", + # dirname=f"{11226}_phantom_walkers_26", + # )], - [(11226, - ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - PARAMS_DEFAULT, - [ - "xrcs.spectra", - "cxff_pi.ti", - "cxff_tws_c.ti", - "ts.ne", - "ts.te", - ]), - dict( - phantom=True, - tstart=0.07, - tend=0.08, - dt=0.01, - iterations=5000, - nwalkers=100, - stopping_criteria_factor=0.01, - mds_write=True, - plot=True, - run="RUN01", - dirname=f"{11226}_phantom_walkers_100", - )], + # [(11226, + # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + # PARAMS_DEFAULT, + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "cxff_tws_c.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=True, + # tstart=0.07, + # tend=0.08, + # dt=0.01, + # iterations=5000, + # nwalkers=50, + # stopping_criteria_factor=0.01, + # mds_write=True, + # plot=True, + # run="RUN01", + # dirname=f"{11226}_phantom_walkers_50", + # )], + # + # [(11226, + # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + # PARAMS_DEFAULT, + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "cxff_tws_c.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=True, + # tstart=0.07, + # tend=0.08, + # dt=0.01, + # iterations=5000, + # nwalkers=100, + # stopping_criteria_factor=0.01, + # mds_write=True, + # plot=True, + # run="RUN01", + # dirname=f"{11226}_phantom_walkers_100", + # )], # [(11226, @@ -260,29 +260,29 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # dirname=f"{11226}_phantom_moment_001", # )], - [(11226, - ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - PARAMS_DEFAULT, - [ - "xrcs.spectra", - "cxff_pi.ti", - "cxff_tws_c.ti", - "ts.ne", - "ts.te", - ]), - dict( - phantom=False, - tstart=0.07, - tend=0.08, - dt=0.01, - iterations=5000, - stopping_criteria_factor=0.01, - nwalkers=50, - mds_write=True, - plot=True, - run="RUN01", - dirname=f"{11226}_exp", - )], + # [(11226, + # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], + # PARAMS_DEFAULT, + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "cxff_tws_c.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=False, + # tstart=0.07, + # tend=0.08, + # dt=0.01, + # iterations=5000, + # stopping_criteria_factor=0.01, + # nwalkers=50, + # mds_write=True, + # plot=True, + # run="RUN01", + # dirname=f"{11226}_exp", + # )], @@ -306,14 +306,34 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # dirname=f"{11336}_phantom", # )], + [(11089, + ["xrcs", "cxff_tws_c", "ts"], + PARAMS_DEFAULT, + [ + "xrcs.spectra", + "cxff_tws_c.ti", + "ts.ne", + "ts.te", + ]), + dict( + phantom=False, + tstart=0.050, + tend=0.150, + dt=0.01, + iterations=5000, + nwalkers=50, + stopping_criteria_factor=0.01, + mds_write=True, + plot=True, + revisions={}, + run="RUN01", + dirname=f"{11089}_quicktest", + )], # RFX low ti/te pulse - ] - - for info in pulse_info: print(f"pulse: {info[0][0]}") bda_run(*info[0], **info[1]) \ No newline at end of file From 78fd1276054f5b9b498a30680ec1cd9131ccb6d6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 30 Oct 2023 16:08:42 +0000 Subject: [PATCH 238/288] temporarily added channel filters for TS / CXFF_TWS_C --- indica/workflows/bayes_workflow.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 2a38a270..acf14932 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -655,11 +655,19 @@ def process_data(self, model_callable: Callable): opt_data["xrcs.spectra"]["error"] = np.sqrt( opt_data["xrcs.spectra"] + background.std(dim="wavelength") ** 2 ) + # TODO move the channel filtering to the read_data method in filtering = {} if "ts.ne" in opt_data.keys(): opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 + opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel<21) if "ts.te" in opt_data.keys(): opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 + opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel<21) + + if "cxff_tws_c.ti" in opt_data.keys(): + opt_data["cxff_tws_c.ti"] = opt_data["cxff_tws_c.ti"].where(opt_data["cxff_tws_c.ti"].channel==0) + + self.opt_data = opt_data From 0ec72e5d469f775866bce305f13593d6ed388072 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 31 Oct 2023 13:58:22 +0000 Subject: [PATCH 239/288] 11089 run added --- indica/workflows/bda_run.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index 93323d61..9778235c 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -306,9 +306,33 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # dirname=f"{11336}_phantom", # )], + # [(11089, + # ["xrcs", "cxff_tws_c", "ts"], + # PARAMS_DEFAULT, + # [ + # "xrcs.spectra", + # "cxff_tws_c.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=False, + # tstart=0.050, + # tend=0.150, + # dt=0.01, + # iterations=5000, + # nwalkers=50, + # stopping_criteria_factor=0.01, + # mds_write=True, + # plot=True, + # revisions={}, + # run="RUN01", + # dirname=f"{11089}_quicktest", + # )], + [(11089, - ["xrcs", "cxff_tws_c", "ts"], - PARAMS_DEFAULT, + ["xrcs", "cxff_tws_c", "ts"], + PARAMS_DEFAULT, [ "xrcs.spectra", "cxff_tws_c.ti", @@ -322,14 +346,13 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, dt=0.01, iterations=5000, nwalkers=50, - stopping_criteria_factor=0.01, + stopping_criteria_factor=0.002, mds_write=True, plot=True, revisions={}, run="RUN01", - dirname=f"{11089}_quicktest", + dirname=f"{11089}", )], - # RFX low ti/te pulse ] From e39440d1ad632296146b514887ac3a21ee10233a Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 31 Oct 2023 13:58:50 +0000 Subject: [PATCH 240/288] expanded wped prior for fitting the pedestal --- indica/workflows/bayes_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index acf14932..a1090230 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -68,7 +68,7 @@ "Ne_prof.y0": get_uniform(2e19, 4e20), "Ne_prof.y1": get_uniform(1e18, 2e19), "Ne_prof.y0/Ne_prof.y1": lambda x1, x2: np.where((x1 > x2 * 2), 1, 0), - "Ne_prof.wped": get_uniform(2, 6), + "Ne_prof.wped": loguniform(2, 20), "Ne_prof.wcenter": get_uniform(0.2, 0.4), "Ne_prof.peaking": get_uniform(1, 4), "Nimp_prof.y0": loguniform(1e16, 1e18), @@ -85,7 +85,7 @@ ), # impurity always more peaked "Te_prof.y0": get_uniform(1000, 5000), - "Te_prof.wped": get_uniform(1, 6), + "Te_prof.wped": get_uniform(1, 8), "Te_prof.wcenter": get_uniform(0.2, 0.4), "Te_prof.peaking": get_uniform(1, 4), # "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode From 70578c8c3ca95f111adf5b1b14c7c42dbb593106 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 2 Nov 2023 10:35:04 +0000 Subject: [PATCH 241/288] Test for walker moves --- indica/workflows/bda_run.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index 9778235c..686d9b38 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -98,7 +98,7 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, ) optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=nwalkers, iterations=iterations, - sample_method="high_density", starting_samples=100, burn_frac=0.20, + sample_method="high_density", starting_samples=200, burn_frac=0.20, stopping_criteria="mode", stopping_criteria_factor=stopping_criteria_factor, stopping_criteria_debug=True, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) @@ -341,17 +341,17 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, ]), dict( phantom=False, - tstart=0.050, - tend=0.150, + tstart=0.09, + tend=0.10, dt=0.01, iterations=5000, nwalkers=50, - stopping_criteria_factor=0.002, + stopping_criteria_factor=0.005, mds_write=True, plot=True, revisions={}, - run="RUN01", - dirname=f"{11089}", + run="TEST", + dirname=f"{11089}_DEWALKER", )], # RFX low ti/te pulse ] From 539821122ca3bc4b3367b5c19630afe4635e6ebc Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 2 Nov 2023 10:46:23 +0000 Subject: [PATCH 242/288] testing DE-MCMC --- indica/workflows/bayes_workflow.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index a1090230..0cacc878 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -85,7 +85,7 @@ ), # impurity always more peaked "Te_prof.y0": get_uniform(1000, 5000), - "Te_prof.wped": get_uniform(1, 8), + "Te_prof.wped": get_uniform(1, 6), "Te_prof.wcenter": get_uniform(0.2, 0.4), "Te_prof.peaking": get_uniform(1, 4), # "Ti_prof.y0/Te_prof.y0": lambda x1, x2: np.where(x1 > x2, 1, 0), # hot ion mode @@ -761,7 +761,7 @@ class EmceeOptimiser(OptimiserContext): def init_optimiser(self, blackbox_func: Callable): ndim = len(self.optimiser_settings.param_names) - self.move = [(emcee.moves.StretchMove(), 0.9), (emcee.moves.DEMove(), 0.1)] + self.move = [(emcee.moves.StretchMove(), 0.0), (emcee.moves.DEMove(), 0.7), (emcee.moves.DESnookerMove(), 0.3)] self.optimiser = emcee.EnsembleSampler( self.optimiser_settings.nwalkers, ndim, @@ -1004,8 +1004,8 @@ def __call__( if __name__ == "__main__": pulse = 11336 - tstart = 0.07 - tend = 0.08 + tstart = 0.05 + tend = 0.07 dt = 0.01 diagnostics = [ From 4cffa96e8bebb7520108f3574858512beb735b7e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 2 Nov 2023 13:30:43 +0000 Subject: [PATCH 243/288] Full 11089 run with DEMOVES --- indica/workflows/bda_run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index 686d9b38..848ebe92 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -341,16 +341,16 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, ]), dict( phantom=False, - tstart=0.09, - tend=0.10, + tstart=0.05, + tend=0.12, dt=0.01, iterations=5000, nwalkers=50, - stopping_criteria_factor=0.005, + stopping_criteria_factor=0.004, mds_write=True, plot=True, revisions={}, - run="TEST", + run="RUN01", dirname=f"{11089}_DEWALKER", )], # RFX low ti/te pulse From f4726e4661a65fa3bb3becc508bbdfd77b3b49db Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 3 Nov 2023 09:29:50 +0000 Subject: [PATCH 244/288] testing without emcee move.DESnookerMove --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 0cacc878..a6ddd8bd 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -761,7 +761,7 @@ class EmceeOptimiser(OptimiserContext): def init_optimiser(self, blackbox_func: Callable): ndim = len(self.optimiser_settings.param_names) - self.move = [(emcee.moves.StretchMove(), 0.0), (emcee.moves.DEMove(), 0.7), (emcee.moves.DESnookerMove(), 0.3)] + self.move = [(emcee.moves.StretchMove(), 0.0), (emcee.moves.DEMove(), 1.0), (emcee.moves.DESnookerMove(), 0.0)] self.optimiser = emcee.EnsembleSampler( self.optimiser_settings.nwalkers, ndim, From 703999154c499011a4a1cedb75cab5613ffc78df Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 3 Nov 2023 09:30:29 +0000 Subject: [PATCH 245/288] Testing without DESnookerMove --- indica/workflows/bda_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index 848ebe92..42605fb0 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -98,7 +98,7 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, ) optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=nwalkers, iterations=iterations, - sample_method="high_density", starting_samples=200, burn_frac=0.20, + sample_method="high_density", starting_samples=300, burn_frac=0.20, stopping_criteria="mode", stopping_criteria_factor=stopping_criteria_factor, stopping_criteria_debug=True, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) @@ -346,12 +346,12 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, dt=0.01, iterations=5000, nwalkers=50, - stopping_criteria_factor=0.004, + stopping_criteria_factor=0.002, mds_write=True, plot=True, revisions={}, run="RUN01", - dirname=f"{11089}_DEWALKER", + dirname=f"{11089}_DEWALKER_NOSNOOKER", )], # RFX low ti/te pulse ] From aa88c264e1a328fbb68143ee13d2f5f5a85b2265 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Tue, 5 Dec 2023 13:03:16 +0000 Subject: [PATCH 246/288] fixed rho_poloidal -> rhop --- indica/workflows/abstract_bayes_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index b98b76f3..270e6808 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -129,8 +129,8 @@ def _build_result_dict(self, ): ) result["PROFILES"] = { - "RHO_POLOIDAL": self.plasma_context.plasma.rho, - "RHO_TOR": self.plasma_context.plasma.equilibrium.rhotor.interp(t=self.plasma_context.plasma.t), + "RHOP": self.plasma_context.plasma.rho, + "RHOT": self.plasma_context.plasma.equilibrium.rhotor.interp(t=self.plasma_context.plasma.t), "NE": self.blobs["electron_density"].median(dim="index"), "NI": self.blobs["ion_density"] .sel(element=self.plasma_context.plasma_settings.main_ion) From d3b1bd4ddf9c0638a59043480815feec8e68a3f8 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 7 Dec 2023 09:50:56 +0000 Subject: [PATCH 247/288] removed wrapper of kernels --- indica/operators/gpr_fit.py | 169 ++++++++++++++---------------------- 1 file changed, 65 insertions(+), 104 deletions(-) diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index 40c15e92..de46d077 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -1,4 +1,5 @@ import getpass +from pathlib import Path import matplotlib.pyplot as plt import numpy as np @@ -8,7 +9,6 @@ from xarray import DataArray from indica.readers.read_st40 import ReadST40 -from indica.utilities import save_figure from indica.utilities import set_axis_sci from indica.utilities import set_plot_colors from indica.utilities import set_plot_rcparams @@ -16,65 +16,12 @@ FIG_PATH = f"/home/{getpass.getuser()}/figures/Indica/profile_fits/" CMAP, COLORS = set_plot_colors() - -def choose_kernel(kernel: str): - """ - Wrapper to return GPR Kernel function selected by name - - Parameters - ---------- - kernel - Name of kernel - - Returns - ------- - Kernel function for GPR call - - """ - kernels = { - "RBF_noise": RadialBasisFunction_WithNoise(), - "RBF": RadialBasisFunction_NoNoise(), - } - - if kernel in kernels.keys(): - return kernels[kernel] - else: - raise ValueError - - -def RadialBasisFunction_NoNoise(length_scale=0.2, dlength_scale=0.001, **kwargs): - kernel = 1.0 * kernels.RBF( - length_scale=length_scale, - length_scale_bounds=( - length_scale - dlength_scale, - length_scale + dlength_scale, - ), - ) - return kernel - - -def RadialBasisFunction_WithNoise( - length_scale=0.2, dlength_scale=0.001, noise_level=10, dnoise_level=10, **kwargs -): - kernel = 1.0 * kernels.RBF( - length_scale=length_scale, - length_scale_bounds=( - length_scale - dlength_scale, - length_scale + dlength_scale, - ), - ) + kernels.WhiteKernel( - noise_level=noise_level, - noise_level_bounds=(noise_level - dnoise_level, noise_level + dnoise_level), - ) - return kernel - - def gpr_fit( x: np.array, y: np.array, y_err: np.array, x_fit: np.array, - kernel_name: str = "RBF_noise", + kernel: kernels.Kernel, ): """ Run GPR fit given input data and desired x-grid @@ -90,7 +37,7 @@ def gpr_fit( x_fit x-axis for fitting kernel - Kernel name + Kernel used Returns ------- @@ -98,8 +45,6 @@ def gpr_fit( """ - kernel = choose_kernel(kernel_name) - isort = np.argsort(x) _x = np.sort(x) _y = np.interp(_x, x[isort], y[isort]) @@ -117,7 +62,7 @@ def gpr_fit( _x_fit = x_fit.reshape(-1, 1) y_fit, y_fit_err = gpr.predict(_x_fit, return_std=True) - return y_fit, y_fit_err + return y_fit, y_fit_err, gpr def plot_gpr_fit( @@ -153,34 +98,26 @@ def plot_gpr_fit( plt.plot(x_data, data, marker="o", linestyle="", color=color, label=label) plt.xlabel(xlabel) plt.ylabel(ylabel) - plt.title(f"{title} t = {data.t.values:.3f} s") + plt.title(title) set_axis_sci() if len(label) > 0: plt.legend() if save_fig: - save_figure(FIG_PATH, f"{fig_name}_GPR_fit_{data.t:.3f}_s", save_fig=save_fig) - - -def example_run( - pulse: int = 10619, - tstart=0.06, - tend=0.1, - kernel_name: str = "RBF_noise", - plot=True, - save_fig=False, - xdim: str = "R", - split="LFS", - virtual_obs=True, - quant = "te", + plt.savefig(FIG_PATH + f"{fig_name}_GPR_t:{data.t.values:.3f}.png", bbox_inches="tight") -): +def read_ts(pulse, tstart, tend, quant, split=""): st40 = ReadST40(pulse, tstart, tend) st40(instruments=["ts", "efit"]) rmag = st40.binned_data["efit"]["rmag"] data = st40.raw_data["ts"][quant] + data["quantity"] = quant + data["pulse"] = pulse + + data = data[(data.t >= tstart) & (data.t <= tend)] + data.transform.set_equilibrium(st40.equilibrium) data.transform.convert_to_rho_theta(t=data.t) data["rho"] = data.transform.rho @@ -189,20 +126,15 @@ def example_run( if quant == "ne": data.values = data.values * 1e-19 data["error"] = data.error * 1e-19 - units = "n19" - else: + data["unit"] = "n19" + elif quant == "te": data.values = data.values * 1e-3 data["error"] = data.error * 1e-3 - units = "keV" - - if xdim == "R": - x_bounds = data.transform._machine_dims[0] + data["unit"] = "keV" else: - x_bounds = (0, 1.2) + raise ValueError(f"Unknown data quantity: {quant}") - data = data[(data.t >= tstart) & (data.t <= tend)] rmag = rmag[0] - if split == "HFS": data = data.where(data.R < rmag) elif split == "LFS": @@ -210,10 +142,36 @@ def example_run( else: data = data + return data + +def gpr_fit_ts( + data: xr.DataArray, + kernel: kernels.Kernel, + xdim: str = "rho", + + virtual_obs=True, + x_bounds=None, + virtual_points=None, + plot = False, + save_fig = False, +): + if x_bounds is None: + if xdim is "rho": + x_bounds = (0, 1.3) + elif xdim is "R": + x_bounds = (0.1, 0.9) + + if virtual_points is None: + if xdim is "rho": + virtual_points = [(-0.2, lambda y: np.nanmax(y)), (1.2, lambda y: 0), ] + elif xdim is "R": + virtual_points = [(0, lambda y: 0), (0.9, lambda y: 0)] + x_fit = np.linspace(x_bounds[0], x_bounds[1], 1000) - # dx = x_fit[1] - x_fit[0] y_fit = [] y_fit_err = [] + gpr = [] + for t in data.t: if "t" in data.__getattr__(xdim).dims: x = data.__getattr__(xdim).sel(t=t).values @@ -223,51 +181,54 @@ def example_run( y_err = data.error.sel(t=t).values if virtual_obs: - x = np.insert(x, [0, x.size], [x_bounds[0], x_bounds[1]]) - if xdim == "rho": - y = np.insert(y, [0, y.size], [np.nanmax(y), 0]) - else: - y = np.insert(y, [0, y.size], [0, 0]) + num_vo = len(virtual_points) + x = np.insert(x, [i for i in range(num_vo)], [virtual_point[0] for virtual_point in virtual_points]) + y = np.insert(y, [i for i in range(num_vo)], [virtual_point[1](y) for virtual_point in virtual_points]) + y_err = np.insert(y_err, [i for i in range(num_vo)], [0.001 for i in range(num_vo)]) - y_err = np.insert(y_err, [0, y_err.size], [0.01, 0.01]) - - _y_fit, _y_fit_err = gpr_fit( + _y_fit, _y_fit_err, _gpr = gpr_fit( x, y, y_err, x_fit, - kernel_name=kernel_name, + kernel, ) y_fit.append(DataArray(_y_fit, coords=[(xdim, x_fit)])) y_fit_err.append(DataArray(_y_fit_err, coords=[(xdim, x_fit)])) + gpr.append(_gpr) fit = xr.concat(y_fit, "t").assign_coords(t=data.t) fit_err = xr.concat(y_fit_err, "t").assign_coords(t=data.t) if plot or save_fig: plt.ioff() - fig_name = f"{pulse}_TS_{quant}" - for tplot in data.t.values: + Path(FIG_PATH).mkdir(parents=True, exist_ok=True) + + + for idx, tplot in enumerate(data.t.values): plot_gpr_fit( data.sel(t=tplot).swap_dims({"channel":xdim}), fit.sel(t=tplot), fit_err.sel(t=tplot), - ylabel=f"{quant} ({units})", + ylabel=f"{data.quantity.values} ({data.unit.values})", xlabel=f"{xdim}", - title=str(st40.pulse), - fig_name=f"{fig_name}_vs_R", + title=f"{data.pulse.values}\nOptimimum: {gpr[idx].kernel_}", + fig_name=f"{data.pulse.values}_TS.{data.quantity.values}", save_fig=save_fig, ) if not save_fig: plt.show() - return data, fit, fit_err + return fit, fit_err if __name__ == "__main__": + kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.1, 0.5), length_scale_bounds=(1, 2.0)) + kernels.WhiteKernel(noise_level_bounds=(0.01, 10)) + # kernel = kernels.RBF(length_scale_bounds=(0.1, 1.0)) + kernels.WhiteKernel(noise_level_bounds=(0.01, 10)) + + quant = "ne" + + data = read_ts(pulse = 11417, tstart=0.05, tend=0.15, quant=quant, split = "LFS",) + gpr_fit_ts(data=data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) - # example_run(pulse=11314, xdim="rho", split="HFS", virtual_obs=True, kernel_name="RBF_noise", quant="ne") - example_run(pulse=11314, xdim="rho", split="LFS", virtual_obs=True, kernel_name="RBF_noise", quant="ne") - # example_run(pulse=11314, xdim="rho", split="HFS", virtual_obs=True, kernel_name="RBF_noise", quant="te") - # example_run(pulse=11314, xdim="rho", split="LFS", virtual_obs=True, kernel_name="RBF_noise", quant="te") From c3e4a36023b155c67799c55fa7f313173f125cdd Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 7 Dec 2023 10:24:17 +0000 Subject: [PATCH 248/288] dt added to read_ts --- indica/operators/gpr_fit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index de46d077..a608067c 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -107,8 +107,8 @@ def plot_gpr_fit( plt.savefig(FIG_PATH + f"{fig_name}_GPR_t:{data.t.values:.3f}.png", bbox_inches="tight") -def read_ts(pulse, tstart, tend, quant, split=""): - st40 = ReadST40(pulse, tstart, tend) +def read_ts(pulse, tstart, tend, dt, quant, split=""): + st40 = ReadST40(pulse, tstart, tend, dt) st40(instruments=["ts", "efit"]) rmag = st40.binned_data["efit"]["rmag"] @@ -229,6 +229,6 @@ def gpr_fit_ts( quant = "ne" - data = read_ts(pulse = 11417, tstart=0.05, tend=0.15, quant=quant, split = "LFS",) + data = read_ts(pulse = 11417, tstart=0.05, tend=0.15, dt=0.01, quant=quant, split = "LFS",) gpr_fit_ts(data=data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) From 5923ea6cae1fe30e50d15be292108cfd34d54ed8 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 7 Dec 2023 10:58:52 +0000 Subject: [PATCH 249/288] using binned data for fits instead of raw --- indica/operators/gpr_fit.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index a608067c..28f8e907 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -112,12 +112,10 @@ def read_ts(pulse, tstart, tend, dt, quant, split=""): st40(instruments=["ts", "efit"]) rmag = st40.binned_data["efit"]["rmag"] - data = st40.raw_data["ts"][quant] + data = st40.binned_data["ts"][quant] data["quantity"] = quant data["pulse"] = pulse - data = data[(data.t >= tstart) & (data.t <= tend)] - data.transform.set_equilibrium(st40.equilibrium) data.transform.convert_to_rho_theta(t=data.t) data["rho"] = data.transform.rho From 9a1420bb53b9ec9d0916fba434a476144408db92 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 7 Dec 2023 14:07:39 +0000 Subject: [PATCH 250/288] hack to prevent dtype interpolation error --- indica/equilibrium.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/equilibrium.py b/indica/equilibrium.py index 04020285..a66df726 100644 --- a/indica/equilibrium.py +++ b/indica/equilibrium.py @@ -575,7 +575,7 @@ def flux_coords( z_x_point = self.zx # TODO: rho and theta dimensions not in the same order... - rho = rho.interp(R=_R, z=_z) + rho = rho.astype("float64").interp(R=_R, z=_z) theta = np.arctan2( _z - z_ax, _R - R_ax, From c89eed8a43b08a8b1dad58017255ccebf68f4315 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 7 Dec 2023 14:08:21 +0000 Subject: [PATCH 251/288] closing plots after use --- indica/operators/gpr_fit.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index 28f8e907..da1eeb4d 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -199,10 +199,7 @@ def gpr_fit_ts( fit_err = xr.concat(y_fit_err, "t").assign_coords(t=data.t) if plot or save_fig: - plt.ioff() Path(FIG_PATH).mkdir(parents=True, exist_ok=True) - - for idx, tplot in enumerate(data.t.values): plot_gpr_fit( data.sel(t=tplot).swap_dims({"channel":xdim}), @@ -218,6 +215,7 @@ def gpr_fit_ts( if not save_fig: plt.show() + plt.close("all") return fit, fit_err From b7fd1a7537fc7436352d1f2d470190967c36f620 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 7 Dec 2023 14:08:49 +0000 Subject: [PATCH 252/288] set_ts_profiles added to plasma_context --- indica/workflows/bayes_workflow.py | 136 ++++++++++++----------------- 1 file changed, 58 insertions(+), 78 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index a6ddd8bd..0e0c0673 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -11,6 +11,7 @@ from scipy.stats import loguniform, describe import xarray as xr import pandas as pd +from sklearn.gaussian_process import kernels from indica.bayesblackbox import BayesBlackBox, ln_prior from indica.bayesblackbox import get_uniform @@ -24,11 +25,11 @@ from indica.models.thomson_scattering import ThomsonScattering from indica.models.thomson_scattering import ts_transform_example from indica.models.plasma import Plasma +from indica.operators.gpr_fit import read_ts, gpr_fit_ts from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow from indica.workflows.bayes_plots import plot_bayes_result from indica.writers.bda_tree import create_nodes from indica.writers.bda_tree import write_nodes -from indica.readers.read_st40 import ReadST40 from indica.equilibrium import Equilibrium from indica.equilibrium import fake_equilibrium from indica.readers.read_st40 import ReadST40 @@ -394,55 +395,26 @@ def save_phantom_profiles(self, kinetic_profiles=None, phantoms=None): } self.phantom_profiles = phantom_profiles - # - # def _init_fast_particles(self, run="RUN602", ): - # - # st40_code = ReadST40(self.astra_pulse_range + self.pulse, self.tstart-self.dt, self.tend+self.dt, dt=self.dt, tree="astra") - # astra_data = st40_code.get_raw_data("", "astra", run) - # - # if self.astra_equilibrium: - # self.equilibrium = Equilibrium(astra_data) - # self.plasma.equilibrium = self.equilibrium - # - # st40_code.bin_data_in_time(["astra"], self.tstart, self.tend, self.dt) - # code_data = st40_code.binned_data["astra"] - # Nf = ( - # code_data["nf"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - # * 1.0e19 - # ) - # self.plasma.fast_density.values = Nf.values - # Nn = ( - # code_data["nn"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - # * 1.0e19 - # ) - # self.plasma.neutral_density.values = Nn.values - # Pblon = code_data["pblon"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - # self.plasma.pressure_fast_parallel.values = Pblon.values - # Pbper = code_data["pbper"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.t) - # self.plasma.pressure_fast_perpendicular.values = Pbper.values - # self.astra_data = code_data - # - # if self.set_ts_profiles: - # overwritten_params = [param for param in self.param_names if any(xs in param for xs in ["Te", "Ne"])] - # if any(overwritten_params): - # raise ValueError(f"Te/Ne set by TS but then the following params overwritten: {overwritten_params}") - # Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 - # self.plasma.Te_prof = lambda: Te.values - # Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 - # self.plasma.Ne_prof = lambda: Ne.values - # - # if self.set_all_profiles: - # overwritten_params = [param for param in self.param_names if any(xs in param for xs in ["Te", "Ti", "Ne", "Nimp"])] - # if any(overwritten_params): - # raise ValueError(f"Te/Ne set by TS but then the following params overwritten: {overwritten_params}") - # Te = code_data["te"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 - # self.plasma.Te_prof = lambda: Te.values - # Ne = code_data["ne"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 - # self.plasma.Ne_prof = lambda: Ne.values - # Ti = code_data["ti"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e3 - # self.plasma.Ti_prof = lambda: Ti.values - # Nimp = code_data["niz1"].interp(rho_poloidal=self.plasma.rho, t=self.plasma.time_to_calculate) * 1e19 - # self.plasma.Nimp_prof = lambda: Nimp.values + + def fit_ts_profile(self, pulse, tstart, tend, dt, split="LFS", quant="ne"): + kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.1, 0.5), + length_scale_bounds=(1, 2.0)) + kernels.WhiteKernel( + noise_level_bounds=(0.01, 10)) + + data = read_ts(pulse=pulse, tstart=tstart, tend=tend, dt=dt, quant=quant, split=split, ) + fit, _ = gpr_fit_ts(data=data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) + fit = xr.where(fit < 0, 0, fit) + return fit + + def set_ts_profiles(self, pulse, tstart, tend, dt, split="LFS"): + + ne_fit = self.fit_ts_profile(pulse, tstart, tend, dt, quant="ne", split=split) * 1e19 + te_fit = self.fit_ts_profile(pulse, tstart, tend, dt, quant="te", split=split) * 1e3 + + self.plasma.electron_density.loc[dict()] = ne_fit.interp(rho=self.plasma.rho) + self.plasma.electron_temperature.loc[dict()] = te_fit.interp(rho=self.plasma.rho) + + @dataclass @@ -458,6 +430,7 @@ class ModelSettings: "xrcs": {"pixel_offset": -4.0} }) + @dataclass class ModelContext: model_settings: ModelSettings @@ -480,7 +453,6 @@ def __post_init__(self): if diag not in self.model_settings.call_kwargs.keys(): self.model_settings.call_kwargs[diag] = {} - def update_model_kwargs(self, data: dict): if "xrcs" in data.keys(): self.model_settings.init_kwargs["xrcs"]["window"] = data["xrcs"]["spectra"].wavelength.values @@ -658,15 +630,14 @@ def process_data(self, model_callable: Callable): # TODO move the channel filtering to the read_data method in filtering = {} if "ts.ne" in opt_data.keys(): opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 - opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel<21) + opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel < 21) if "ts.te" in opt_data.keys(): opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 - opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel<21) + opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel < 21) if "cxff_tws_c.ti" in opt_data.keys(): - opt_data["cxff_tws_c.ti"] = opt_data["cxff_tws_c.ti"].where(opt_data["cxff_tws_c.ti"].channel==0) - + opt_data["cxff_tws_c.ti"] = opt_data["cxff_tws_c.ti"].where(opt_data["cxff_tws_c.ti"].channel == 0) self.opt_data = opt_data @@ -925,15 +896,6 @@ def __init__( self.tend = tend self.dt = dt - self.plasma_context.init_plasma(equilibrium=self.data_context.equilibrium, tstart=self.tstart, tend=self.tend, - dt=self.dt, ) - self.plasma_context.save_phantom_profiles(phantoms=self.data_context.phantoms) - - self.model_context.update_model_kwargs(self.data_context.binned_data) - self.model_context.init_models() - - self.data_context.process_data(self.model_context._build_bckc, ) - self.blackbox = BayesBlackBox(data=self.data_context.opt_data, plasma_context=self.plasma_context, model_context=self.model_context, @@ -1005,30 +967,35 @@ def __call__( if __name__ == "__main__": pulse = 11336 tstart = 0.05 - tend = 0.07 + tend = 0.06 dt = 0.01 diagnostics = [ - # "xrcs", + "xrcs", # "efit", # "smmh1", # "cxff_pi", - # "cxff_tws_c", + "cxff_tws_c", "ts", ] # diagnostic_quantities opt_quant = [ - # "xrcs.spectra", + "xrcs.spectra", + "cxff_tws_c.ti", # "efit.wp", - "ts.te" + "ts.te", + "ts.ne", ] opt_params = [ - "Te_prof.y0", - "Te_prof.peaking", - "Te_prof.wped", - "Te_prof.wcenter", - - # "Ti_prof.y0", + # "Te_prof.y0", + # "Te_prof.peaking", + # "Te_prof.wped", + # "Te_prof.wcenter", + "Ti_prof.y0", + "Ti_prof.peaking", + "Ti_prof.wped", + "Ti_prof.wcenter", + "Nimp_prof.y0", # "Ne_prof.y0", ] @@ -1043,7 +1010,7 @@ def __call__( # data_context = PhantomData(pulse=pulse, diagnostics=diagnostics, # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context = ExpData(pulse=pulse, diagnostics=diagnostics, - tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context.read_data() plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), @@ -1059,7 +1026,20 @@ def __call__( model_settings=model_settings, ) - optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=100, + # Initialise context objects + plasma_context.init_plasma(equilibrium=data_context.equilibrium, tstart=tstart, tend=tend, + dt=dt, ) + + plasma_context.set_ts_profiles(pulse, tstart, tend, dt, split="HFS") + + plasma_context.save_phantom_profiles(phantoms=data_context.phantoms) + + model_context.update_model_kwargs(data_context.binned_data) + model_context.init_models() + + data_context.process_data(model_context._build_bckc, ) + + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=20, sample_method="high_density", starting_samples=100, burn_frac=0.20, stopping_criteria="mode", stopping_criteria_factor=0.01, priors=bayes_settings.priors) @@ -1070,4 +1050,4 @@ def __call__( optimiser_context=optimiser_context, plasma_context=plasma_context, model_context=model_context) - workflow(pulse_to_write=25000000, run="RUN01", mds_write=True, plot=True, filepath="./results/test/") + workflow(pulse_to_write=25000000, run="RUN01", mds_write=False, plot=True, filepath="./results/test/") From cb8d99c720f1e8ec40cfcb801248d58a924d1f13 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 7 Dec 2023 16:07:19 +0000 Subject: [PATCH 253/288] update_profiles now only updates profiles which match parameters provided --- indica/models/plasma.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index b119af32..d59d929d 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -520,7 +520,7 @@ def update_profiles( Update plasma profiles with profile parameters i.e. {"Ne_prof.y0":1e19} -> Ne_prof.y0 """ - profile_prefixs: list = [ + profile_prefixes: list = [ "Te_prof", "Ti_prof", "Ne_prof", @@ -528,7 +528,7 @@ def update_profiles( "Vrot_prof", ] for param, value in parameters.items(): - _prefix = [pref for pref in profile_prefixs if pref in param] + _prefix = [pref for pref in profile_prefixes if pref in param] if _prefix: prefix: str = _prefix[0] key = param.replace(prefix + ".", "") @@ -538,14 +538,21 @@ def update_profiles( else: raise ValueError(f"parameter: {key} not found in {prefix}") - for key in [ - "electron_density", - "electron_temperature", - "ion_temperature", - "toroidal_rotation", - "impurity_density", - ]: - self.assign_profiles(key, t=self.time_to_calculate) + # Only update profiles which are given in parameters + parameter_prefixes = [key.split(".")[0] for key in parameters.keys()] + profile_names = set(parameter_prefixes) & set(profile_prefixes) + + profile_mapping = { + "Te_prof": "electron_temperature", + "Ti_prof": "ion_temperature", + "Ne_prof": "electron_density", + "Vrot_prof": "toroidal_rotation", + "Nimp_prof": "impurity_density", + "Nh_prof": "neutral_density", + } + + for key in profile_names: + self.assign_profiles(profile_mapping[key], t=self.time_to_calculate) @property def time_to_calculate(self): From f5aadf3577934b28faacf6b3eb2cf1a0c6389293 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 8 Dec 2023 10:35:12 +0000 Subject: [PATCH 254/288] __main__ params adjusted --- indica/operators/gpr_fit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index da1eeb4d..ca0dc938 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -161,7 +161,7 @@ def gpr_fit_ts( if virtual_points is None: if xdim is "rho": - virtual_points = [(-0.2, lambda y: np.nanmax(y)), (1.2, lambda y: 0), ] + virtual_points = [(-0.2, lambda y: np.nanmax(y)), (2.0, lambda y: 0), ] elif xdim is "R": virtual_points = [(0, lambda y: 0), (0.9, lambda y: 0)] @@ -220,11 +220,11 @@ def gpr_fit_ts( if __name__ == "__main__": - kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.1, 0.5), length_scale_bounds=(1, 2.0)) + kernels.WhiteKernel(noise_level_bounds=(0.01, 10)) + kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.5, 1.0), length_scale_bounds=(0.4, 0.7)) + kernels.WhiteKernel(noise_level_bounds=(0.01, 10)) # kernel = kernels.RBF(length_scale_bounds=(0.1, 1.0)) + kernels.WhiteKernel(noise_level_bounds=(0.01, 10)) quant = "ne" - data = read_ts(pulse = 11417, tstart=0.05, tend=0.15, dt=0.01, quant=quant, split = "LFS",) + data = read_ts(pulse = 11089, tstart=0.05, tend=0.15, dt=0.01, quant=quant, split = "LFS",) gpr_fit_ts(data=data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) From fc577a2ec52aeefe790f581d0192dceb1a1f83c6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 8 Dec 2023 12:33:08 +0000 Subject: [PATCH 255/288] reformatting __main__ to run mock example --- indica/workflows/bayes_workflow.py | 44 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 0e0c0673..affd9d08 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -397,13 +397,13 @@ def save_phantom_profiles(self, kinetic_profiles=None, phantoms=None): def fit_ts_profile(self, pulse, tstart, tend, dt, split="LFS", quant="ne"): - kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.1, 0.5), - length_scale_bounds=(1, 2.0)) + kernels.WhiteKernel( + kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.5, 1.0), + length_scale_bounds=(0.4, 0.7)) + kernels.WhiteKernel( noise_level_bounds=(0.01, 10)) data = read_ts(pulse=pulse, tstart=tstart, tend=tend, dt=dt, quant=quant, split=split, ) fit, _ = gpr_fit_ts(data=data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) - fit = xr.where(fit < 0, 0, fit) + fit = xr.where(fit < 0, 1e-10, fit) return fit def set_ts_profiles(self, pulse, tstart, tend, dt, split="LFS"): @@ -630,11 +630,11 @@ def process_data(self, model_callable: Callable): # TODO move the channel filtering to the read_data method in filtering = {} if "ts.ne" in opt_data.keys(): opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 - opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel < 21) + # opt_data["ts.ne"] = opt_data["ts.ne"].where(opt_data["ts.ne"].channel < 21) if "ts.te" in opt_data.keys(): opt_data["ts.ne"]["error"] = opt_data["ts.ne"].max(dim="channel") * 0.05 - opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel < 21) + # opt_data["ts.te"] = opt_data["ts.te"].where(opt_data["ts.te"].channel < 21) if "cxff_tws_c.ti" in opt_data.keys(): opt_data["cxff_tws_c.ti"] = opt_data["cxff_tws_c.ti"].where(opt_data["cxff_tws_c.ti"].channel == 0) @@ -965,7 +965,7 @@ def __call__( if __name__ == "__main__": - pulse = 11336 + pulse = None tstart = 0.05 tend = 0.06 dt = 0.01 @@ -987,15 +987,19 @@ def __call__( "ts.ne", ] opt_params = [ - # "Te_prof.y0", + "Ne_prof.y0", + "Te_prof.y0", # "Te_prof.peaking", # "Te_prof.wped", # "Te_prof.wcenter", "Ti_prof.y0", - "Ti_prof.peaking", - "Ti_prof.wped", - "Ti_prof.wcenter", - "Nimp_prof.y0", + # "Ti_prof.peaking", + # "Ti_prof.wped", + # "Ti_prof.wcenter", + # "Nimp_prof.y0", + # "Nimp_prof.peaking", + # "Nimp_prof.wcenter", + # "Nimp_prof.wped", # "Ne_prof.y0", ] @@ -1005,19 +1009,19 @@ def __call__( data_settings = ReaderSettings(filters={}, revisions={}) # Add general methods for filtering data co-ords to ReadST40 - # data_context = MockData(pulse=pulse, diagnostics=diagnostics, - # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + data_context = MockData(pulse=pulse, diagnostics=diagnostics, + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) # data_context = PhantomData(pulse=pulse, diagnostics=diagnostics, # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) - data_context = ExpData(pulse=pulse, diagnostics=diagnostics, - tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + # data_context = ExpData(pulse=pulse, diagnostics=diagnostics, + # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context.read_data() plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), n_rad=20) plasma_context = PlasmaContext(plasma_settings=plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) - model_settings = ModelSettings(call_kwargs={"xrcs": {"pixel_offset": -4.0}}) + model_settings = ModelSettings(call_kwargs={"xrcs": {"pixel_offset": 0.0}}) model_context = ModelContext(diagnostics=diagnostics, plasma_context=plasma_context, @@ -1030,7 +1034,7 @@ def __call__( plasma_context.init_plasma(equilibrium=data_context.equilibrium, tstart=tstart, tend=tend, dt=dt, ) - plasma_context.set_ts_profiles(pulse, tstart, tend, dt, split="HFS") + # plasma_context.set_ts_profiles(pulse, tstart, tend, dt, split="LFS") plasma_context.save_phantom_profiles(phantoms=data_context.phantoms) @@ -1039,9 +1043,9 @@ def __call__( data_context.process_data(model_context._build_bckc, ) - optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=20, + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=1000, sample_method="high_density", starting_samples=100, burn_frac=0.20, - stopping_criteria="mode", stopping_criteria_factor=0.01, + stopping_criteria="mode", stopping_criteria_factor=0.01, stopping_criteria_debug=True, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) @@ -1050,4 +1054,4 @@ def __call__( optimiser_context=optimiser_context, plasma_context=plasma_context, model_context=model_context) - workflow(pulse_to_write=25000000, run="RUN01", mds_write=False, plot=True, filepath="./results/test/") + workflow(pulse_to_write=43000000, run="RUN01", mds_write=True, plot=True, filepath=f"./results/test/") From 97cad5471eaa868e2da09c7134a34cc388419c62 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 13 Dec 2023 15:02:02 +0000 Subject: [PATCH 256/288] fixed error in zeff calculatio --- indica/models/plasma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index d59d929d..5f59a3e1 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -656,7 +656,7 @@ def zeff(self): # return self.calc_zeff() def calc_zeff(self): - self._zeff = (self.ion_density * self.meanz) ** 2 / self.electron_density + self._zeff = self.ion_density * self.meanz ** 2 / self.electron_density return self._zeff @property From 5e16452bfc662bcad65d9f1939e4ca7f770485b2 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 14 Dec 2023 15:01:41 +0000 Subject: [PATCH 257/288] removed reading ts and added post_process_ts --- indica/operators/gpr_fit.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index ca0dc938..87d0bcf1 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -1,5 +1,6 @@ import getpass from pathlib import Path +from typing import Dict import matplotlib.pyplot as plt import numpy as np @@ -107,14 +108,11 @@ def plot_gpr_fit( plt.savefig(FIG_PATH + f"{fig_name}_GPR_t:{data.t.values:.3f}.png", bbox_inches="tight") -def read_ts(pulse, tstart, tend, dt, quant, split=""): - st40 = ReadST40(pulse, tstart, tend, dt) - st40(instruments=["ts", "efit"]) +def post_process_ts(st40: ReadST40, quant, pulse, split=""): rmag = st40.binned_data["efit"]["rmag"] - data = st40.binned_data["ts"][quant] - data["quantity"] = quant data["pulse"] = pulse + data["quantity"] = quant data.transform.set_equilibrium(st40.equilibrium) data.transform.convert_to_rho_theta(t=data.t) @@ -224,7 +222,14 @@ def gpr_fit_ts( # kernel = kernels.RBF(length_scale_bounds=(0.1, 1.0)) + kernels.WhiteKernel(noise_level_bounds=(0.01, 10)) quant = "ne" + pulse = 11089 + tstart = 0.05 + tend = 0.15 + dt = 0.01 + + st40 = ReadST40(pulse, tstart, tend, dt) + st40(instruments=["ts", "efit"]) - data = read_ts(pulse = 11089, tstart=0.05, tend=0.15, dt=0.01, quant=quant, split = "LFS",) + data = post_process_ts(st40, quant, pulse, split = "LFS",) gpr_fit_ts(data=data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) From 5abc8914c25e40a425c1a6221856bb7d17d3a16b Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 09:28:43 +0000 Subject: [PATCH 258/288] reorganising post_processing args --- indica/operators/gpr_fit.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index 87d0bcf1..c6e44061 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -108,13 +108,13 @@ def plot_gpr_fit( plt.savefig(FIG_PATH + f"{fig_name}_GPR_t:{data.t.values:.3f}.png", bbox_inches="tight") -def post_process_ts(st40: ReadST40, quant, pulse, split=""): - rmag = st40.binned_data["efit"]["rmag"] - data = st40.binned_data["ts"][quant] +def post_process_ts(binned_data, equilibrium, quant, pulse, split=""): + rmag = binned_data["efit"]["rmag"] + data = binned_data["ts"][quant] data["pulse"] = pulse data["quantity"] = quant - data.transform.set_equilibrium(st40.equilibrium) + data.transform.set_equilibrium(equilibrium) data.transform.convert_to_rho_theta(t=data.t) data["rho"] = data.transform.rho @@ -230,6 +230,6 @@ def gpr_fit_ts( st40 = ReadST40(pulse, tstart, tend, dt) st40(instruments=["ts", "efit"]) - data = post_process_ts(st40, quant, pulse, split = "LFS",) + data = post_process_ts(st40.binned_data, st40.equilibrium, quant, pulse, split = "LFS",) gpr_fit_ts(data=data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) From 7f7ac085256265a7cba613ab9f93d1e10d623643 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 09:30:27 +0000 Subject: [PATCH 259/288] R_midplane added to profiles --- indica/workflows/abstract_bayes_workflow.py | 171 ++++++++++++++++---- 1 file changed, 139 insertions(+), 32 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 270e6808..53c2356f 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -129,33 +129,66 @@ def _build_result_dict(self, ): ) result["PROFILES"] = { - "RHOP": self.plasma_context.plasma.rho, - "RHOT": self.plasma_context.plasma.equilibrium.rhotor.interp(t=self.plasma_context.plasma.t), - "NE": self.blobs["electron_density"].median(dim="index"), - "NI": self.blobs["ion_density"] - .sel(element=self.plasma_context.plasma_settings.main_ion) - .median(dim="index"), - "TE": self.blobs["electron_temperature"].median(dim="index"), - "TI": self.blobs["ion_temperature"] - .sel(element=self.plasma_context.plasma_settings.main_ion) - .median(dim="index"), - "NFAST": self.blobs["fast_density"].median(dim="index"), - "NNEUTR": self.blobs["neutral_density"].median(dim="index"), - "NE_ERR": self.blobs["electron_density"].std(dim="index"), - "NI_ERR": self.blobs["ion_density"] - .sel(element=self.plasma_context.plasma_settings.main_ion) - .std(dim="index"), - "TE_ERR": self.blobs["electron_temperature"].std(dim="index"), - "TI_ERR": self.blobs["ion_temperature"] - .sel(element=self.plasma_context.plasma_settings.main_ion) - .std(dim="index"), - "NFAST_ERR": self.blobs["fast_density"].std(dim="index"), - "NNEUTR_ERR": self.blobs["neutral_density"].std(dim="index"), - "ZEFF": self.blobs["zeff"].sum("element").median(dim="index"), - "ZEFF_ERR": self.blobs["zeff"].sum("element").std(dim="index"), + "PSI_NORM":{ + "RHOP": self.plasma_context.plasma.rho, + "RHOT": self.plasma_context.plasma.equilibrium.rhotor.interp(t=self.plasma_context.plasma.t), + "NE": self.blobs["electron_density"].median(dim="index"), + "NI": self.blobs["ion_density"] + .sel(element=self.plasma_context.plasma_settings.main_ion) + .median(dim="index"), + "TE": self.blobs["electron_temperature"].median(dim="index"), + "TI": self.blobs["ion_temperature"] + .sel(element=self.plasma_context.plasma_settings.main_ion) + .median(dim="index"), + "NFAST": self.blobs["fast_density"].median(dim="index"), + "NNEUTR": self.blobs["neutral_density"].median(dim="index"), + "NE_ERR": self.blobs["electron_density"].std(dim="index"), + "NI_ERR": self.blobs["ion_density"] + .sel(element=self.plasma_context.plasma_settings.main_ion) + .std(dim="index"), + "TE_ERR": self.blobs["electron_temperature"].std(dim="index"), + "TI_ERR": self.blobs["ion_temperature"] + .sel(element=self.plasma_context.plasma_settings.main_ion) + .std(dim="index"), + "NFAST_ERR": self.blobs["fast_density"].std(dim="index"), + "NNEUTR_ERR": self.blobs["neutral_density"].std(dim="index"), + "ZEFF": self.blobs["zeff"].sum("element").median(dim="index"), + "ZEFF_ERR": self.blobs["zeff"].sum("element").std(dim="index"), + "ZI": self.blobs["zeff"].sel(element=self.plasma_context.plasma_settings.main_ion).median(dim="index"), + "ZI_ERR": self.blobs["zeff"].sel(element=self.plasma_context.plasma_settings.main_ion).std(dim="index"), + }, + "R_MIDPLANE": { + "RPOS": self.midplane_blobs["electron_temperature"].R, + "ZPOS": self.midplane_blobs["electron_temperature"].z, + "NE": self.midplane_blobs["electron_density"].median(dim="index"), + "NI": self.midplane_blobs["ion_density"] + .sel(element=self.plasma_context.plasma_settings.main_ion) + .median(dim="index"), + "TE": self.midplane_blobs["electron_temperature"].median(dim="index"), + "TI": self.midplane_blobs["ion_temperature"] + .sel(element=self.plasma_context.plasma_settings.main_ion) + .median(dim="index"), + "NFAST": self.midplane_blobs["fast_density"].median(dim="index"), + "NNEUTR": self.midplane_blobs["neutral_density"].median(dim="index"), + "NE_ERR": self.midplane_blobs["electron_density"].std(dim="index"), + "NI_ERR": self.midplane_blobs["ion_density"] + .sel(element=self.plasma_context.plasma_settings.main_ion) + .std(dim="index"), + "TE_ERR": self.midplane_blobs["electron_temperature"].std(dim="index"), + "TI_ERR": self.midplane_blobs["ion_temperature"] + .sel(element=self.plasma_context.plasma_settings.main_ion) + .std(dim="index"), + "NFAST_ERR": self.midplane_blobs["fast_density"].std(dim="index"), + "NNEUTR_ERR": self.midplane_blobs["neutral_density"].std(dim="index"), + "ZEFF": self.midplane_blobs["zeff"].sum("element").median(dim="index"), + "ZEFF_ERR": self.midplane_blobs["zeff"].sum("element").std(dim="index"), + "ZI": self.midplane_blobs["zeff"].sel(element=self.plasma_context.plasma_settings.main_ion).median(dim="index"), + "ZI_ERR": self.midplane_blobs["zeff"].sel(element=self.plasma_context.plasma_settings.main_ion).std(dim="index"), + + }, } - result["PROFILES"] = { - **result["PROFILES"], + result["PROFILES"]["PSI_NORM"] = { + **result["PROFILES"]["PSI_NORM"], **{ f"NIZ{num_imp + 1}": self.blobs["impurity_density"] .sel(element=imp) @@ -163,8 +196,8 @@ def _build_result_dict(self, ): for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } - result["PROFILES"] = { - **result["PROFILES"], + result["PROFILES"]["PSI_NORM"] = { + **result["PROFILES"]["PSI_NORM"], **{ f"NIZ{num_imp + 1}_ERR": self.blobs["impurity_density"] .sel(element=imp) @@ -172,8 +205,8 @@ def _build_result_dict(self, ): for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } - result["PROFILES"] = { - **result["PROFILES"], + result["PROFILES"]["PSI_NORM"] = { + **result["PROFILES"]["PSI_NORM"], **{ f"TIZ{num_imp + 1}": self.blobs["ion_temperature"] .sel(element=imp) @@ -181,8 +214,8 @@ def _build_result_dict(self, ): for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) }, } - result["PROFILES"] = { - **result["PROFILES"], + result["PROFILES"]["PSI_NORM"] = { + **result["PROFILES"]["PSI_NORM"], **{ f"TIZ{num_imp + 1}_ERR": self.blobs["ion_temperature"] .sel(element=imp) @@ -191,6 +224,80 @@ def _build_result_dict(self, ): }, } + result["PROFILES"]["PSI_NORM"] = { + **result["PROFILES"]["PSI_NORM"], + **{ + f"ZIM{num_imp + 1}": self.blobs["zeff"] + .sel(element=imp) + .median(dim="index") + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) + }, + } + result["PROFILES"]["PSI_NORM"] = { + **result["PROFILES"]["PSI_NORM"], + **{ + f"ZIM{num_imp + 1}_ERR": self.blobs["zeff"] + .sel(element=imp) + .std(dim="index") + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) + }, + } + + result["PROFILES"]["R_MIDPLANE"] = { + **result["PROFILES"]["R_MIDPLANE"], + **{ + f"NIZ{num_imp + 1}": self.midplane_blobs["impurity_density"] + .sel(element=imp) + .median(dim="index") + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) + }, + } + result["PROFILES"]["R_MIDPLANE"] = { + **result["PROFILES"]["R_MIDPLANE"], + **{ + f"NIZ{num_imp + 1}_ERR": self.midplane_blobs["impurity_density"] + .sel(element=imp) + .std(dim="index") + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) + }, + } + result["PROFILES"]["R_MIDPLANE"] = { + **result["PROFILES"]["R_MIDPLANE"], + **{ + f"TIZ{num_imp + 1}": self.midplane_blobs["ion_temperature"] + .sel(element=imp) + .median(dim="index") + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) + }, + } + result["PROFILES"]["R_MIDPLANE"] = { + **result["PROFILES"]["R_MIDPLANE"], + **{ + f"TIZ{num_imp + 1}_ERR": self.midplane_blobs["ion_temperature"] + .sel(element=imp) + .std(dim="index") + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) + }, + } + result["PROFILES"]["R_MIDPLANE"] = { + **result["PROFILES"]["R_MIDPLANE"], + **{ + f"ZIM{num_imp + 1}": self.midplane_blobs["zeff"] + .sel(element=imp) + .median(dim="index") + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) + }, + } + result["PROFILES"]["R_MIDPLANE"] = { + **result["PROFILES"]["R_MIDPLANE"], + **{ + f"ZIM{num_imp + 1}_ERR": self.midplane_blobs["zeff"] + .sel(element=imp) + .std(dim="index") + for num_imp, imp in enumerate(self.plasma_context.plasma_settings.impurities) + }, + } + result["PROFILE_STAT"] = { "SAMPLE_IDX": np.arange(self.optimiser_context.optimiser_settings.iterations * (1-self.optimiser_context.optimiser_settings.burn_frac)), From 883546b92b5e10276baa2cff33eab908e94b2ad9 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 09:47:04 +0000 Subject: [PATCH 260/288] gitcommit and user added --- indica/workflows/abstract_bayes_workflow.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/indica/workflows/abstract_bayes_workflow.py b/indica/workflows/abstract_bayes_workflow.py index 53c2356f..1987d9f7 100644 --- a/indica/workflows/abstract_bayes_workflow.py +++ b/indica/workflows/abstract_bayes_workflow.py @@ -2,7 +2,8 @@ from abc import abstractmethod from pathlib import Path import pickle - +import git +import getpass import numpy as np class AbstractBayesWorkflow(ABC): @@ -40,8 +41,8 @@ def _build_inputs_dict(self): result["TIME"] = self.plasma_context.plasma.t result["METADATA"] = { - "GITCOMMIT": "PLACEHOLDER", - "USER": "PLACEHOLDER", + "GITCOMMIT": f"{git.Repo(search_parent_directories=True).head.object.hexsha}", + "USER": f"{getpass.getuser()}", "EQUIL": "PLACEHOLDER", } From 67c1a449c9384a138f6716bf5fb05de1a79e4a7b Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 09:49:00 +0000 Subject: [PATCH 261/288] R_midplane added --- indica/writers/bda_tree.py | 115 ++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 33 deletions(-) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 0a3bc894..b26f4a4d 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -30,40 +30,89 @@ def bda(): "EQUIL": ("TEXT", "Equilibrium used"), }, "PROFILES": { - "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), - "RHOT": ("SIGNAL", "Radial vector, toroidal flux"), - "NE": ("SIGNAL", "Electron density, m^-3"), - "NI": ("SIGNAL", "Ion density, m^-3"), - "TE": ("SIGNAL", "Electron temperature, eV"), - "TI": ("SIGNAL", "Ion temperature of main ion, eV"), - "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1, eV"), - "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), - "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), - "NIZ1": ("SIGNAL", "Density of impurity Z1, m^-3"), - "NIZ2": ("SIGNAL", "Density of impurity Z2, m^-3"), - "NIZ3": ("SIGNAL", "Density of impurity Z3, m^-3"), - "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), - "NFAST": ("SIGNAL", "Density of fast ion, m^-3"), - "ZI": ("SIGNAL", "Average charge of main ion, "), - "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), - "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), - "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), - "ZEFF": ("SIGNAL", "Effective charge, "), - "P": ("SIGNAL", "Pressure,Pa"), - "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), - "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), - "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), - "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), - "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), - "TIZ1_ERR": ("SIGNAL", "Ion temperature of impurity Z1 error, eV"), - "TIZ2_ERR": ("SIGNAL", "Ion temperature of impurity Z2 error, eV"), - "TIZ3_ERR": ("SIGNAL", "Ion temperature of impurity Z3 error, eV"), - "NIZ1_ERR": ("SIGNAL", "Density of impurity Z1 error, m^-3"), - "NIZ2_ERR": ("SIGNAL", "Density of impurity Z2 error, m^-3"), - "NIZ3_ERR": ("SIGNAL", "Density of impurity Z3 error, m^-3"), - "NNEUTR_ERR": ("SIGNAL", "Density of neutral main ion error, m^-3"), - "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), + "PSI_NORM": { + "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "RHOT": ("SIGNAL", "Radial vector, toroidal flux"), + "NE": ("SIGNAL", "Electron density, m^-3"), + "NI": ("SIGNAL", "Ion density, m^-3"), + "TE": ("SIGNAL", "Electron temperature, eV"), + "TI": ("SIGNAL", "Ion temperature of main ion, eV"), + "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1, eV"), + "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), + "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), + "NIZ1": ("SIGNAL", "Density of impurity Z1, m^-3"), + "NIZ2": ("SIGNAL", "Density of impurity Z2, m^-3"), + "NIZ3": ("SIGNAL", "Density of impurity Z3, m^-3"), + "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), + "NFAST": ("SIGNAL", "Density of fast ion, m^-3"), + "ZI": ("SIGNAL", "Average charge of main ion, "), + "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), + "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), + "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), + "ZEFF": ("SIGNAL", "Effective charge, "), + "P": ("SIGNAL", "Pressure,Pa"), + "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), + "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), + "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), + "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), + "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), + "TIZ1_ERR": ("SIGNAL", "Ion temperature of impurity Z1 error, eV"), + "TIZ2_ERR": ("SIGNAL", "Ion temperature of impurity Z2 error, eV"), + "TIZ3_ERR": ("SIGNAL", "Ion temperature of impurity Z3 error, eV"), + "NIZ1_ERR": ("SIGNAL", "Density of impurity Z1 error, m^-3"), + "NIZ2_ERR": ("SIGNAL", "Density of impurity Z2 error, m^-3"), + "NIZ3_ERR": ("SIGNAL", "Density of impurity Z3 error, m^-3"), + "NNEUTR_ERR": ("SIGNAL", "Density of neutral main ion error, m^-3"), + "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), + "ZI_ERR": ("SIGNAL", "Average charge of main ion error, "), + "ZIM1_ERR": ("SIGNAL", "Average charge of impurity IMP1 error, "), + "ZIM2_ERR": ("SIGNAL", "Average charge of impurity IMP2 error, "), + "ZIM3_ERR": ("SIGNAL", "Average charge of impurity IMP3 error, "), + "ZEFF_ERR": ("SIGNAL", "Effective charge error, "), + }, + + "R_MIDPLANE": { + "RPOS": ("NUMERIC", "Major radius position of measurement, m"), + "ZPOS": ("NUMERIC", "Z position of measurement, m"), + "NE": ("SIGNAL", "Electron density, m^-3"), + "NI": ("SIGNAL", "Ion density, m^-3"), + "TE": ("SIGNAL", "Electron temperature, eV"), + "TI": ("SIGNAL", "Ion temperature of main ion, eV"), + "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1, eV"), + "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), + "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), + "NIZ1": ("SIGNAL", "Density of impurity Z1, m^-3"), + "NIZ2": ("SIGNAL", "Density of impurity Z2, m^-3"), + "NIZ3": ("SIGNAL", "Density of impurity Z3, m^-3"), + "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), + "NFAST": ("SIGNAL", "Density of fast ion, m^-3"), + "ZI": ("SIGNAL", "Average charge of main ion, "), + "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), + "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), + "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), + "ZEFF": ("SIGNAL", "Effective charge, "), + "P": ("SIGNAL", "Pressure,Pa"), + "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), + "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), + "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), + "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), + "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), + "TIZ1_ERR": ("SIGNAL", "Ion temperature of impurity Z1 error, eV"), + "TIZ2_ERR": ("SIGNAL", "Ion temperature of impurity Z2 error, eV"), + "TIZ3_ERR": ("SIGNAL", "Ion temperature of impurity Z3 error, eV"), + "NIZ1_ERR": ("SIGNAL", "Density of impurity Z1 error, m^-3"), + "NIZ2_ERR": ("SIGNAL", "Density of impurity Z2 error, m^-3"), + "NIZ3_ERR": ("SIGNAL", "Density of impurity Z3 error, m^-3"), + "NNEUTR_ERR": ("SIGNAL", "Density of neutral main ion error, m^-3"), + "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), + "ZI_ERR": ("SIGNAL", "Average charge of main ion error, "), + "ZIM1_ERR": ("SIGNAL", "Average charge of impurity IMP1 error, "), + "ZIM2_ERR": ("SIGNAL", "Average charge of impurity IMP2 error, "), + "ZIM3_ERR": ("SIGNAL", "Average charge of impurity IMP3 error, "), + "ZEFF_ERR": ("SIGNAL", "Effective charge error, "), + }, }, + "PROFILE_STAT": { "SAMPLE_IDX": ("NUMERIC", "Index of the optimisation samples"), "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), From fededb17bc6b57faaaf0945e51e79b102518a854 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 09:50:29 +0000 Subject: [PATCH 262/288] R_midplane co-ordinate transform added --- indica/workflows/bayes_workflow.py | 90 ++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index affd9d08..8c3ce6d1 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -4,6 +4,7 @@ from abc import abstractmethod from dataclasses import dataclass, field from functools import partial +from copy import deepcopy import emcee import flatdict @@ -25,7 +26,7 @@ from indica.models.thomson_scattering import ThomsonScattering from indica.models.thomson_scattering import ts_transform_example from indica.models.plasma import Plasma -from indica.operators.gpr_fit import read_ts, gpr_fit_ts +from indica.operators.gpr_fit import post_process_ts, gpr_fit_ts from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow from indica.workflows.bayes_plots import plot_bayes_result from indica.writers.bda_tree import create_nodes @@ -395,27 +396,52 @@ def save_phantom_profiles(self, kinetic_profiles=None, phantoms=None): } self.phantom_profiles = phantom_profiles - - def fit_ts_profile(self, pulse, tstart, tend, dt, split="LFS", quant="ne"): + def fit_ts_profile(self, data_context, split="LFS", quant="ne"): kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.5, 1.0), length_scale_bounds=(0.4, 0.7)) + kernels.WhiteKernel( noise_level_bounds=(0.01, 10)) - data = read_ts(pulse=pulse, tstart=tstart, tend=tend, dt=dt, quant=quant, split=split, ) - fit, _ = gpr_fit_ts(data=data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) + processed_data = post_process_ts(deepcopy(data_context.binned_data), data_context.equilibrium, quant, data_context.pulse, split=split, ) + fit, _ = gpr_fit_ts(data=processed_data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) fit = xr.where(fit < 0, 1e-10, fit) return fit - def set_ts_profiles(self, pulse, tstart, tend, dt, split="LFS"): + def set_ts_profiles(self, data_context, split="LFS"): - ne_fit = self.fit_ts_profile(pulse, tstart, tend, dt, quant="ne", split=split) * 1e19 - te_fit = self.fit_ts_profile(pulse, tstart, tend, dt, quant="te", split=split) * 1e3 + ne_fit = self.fit_ts_profile(data_context, quant="ne", split=split) * 1e19 + te_fit = self.fit_ts_profile(data_context, quant="te", split=split) * 1e3 self.plasma.electron_density.loc[dict()] = ne_fit.interp(rho=self.plasma.rho) self.plasma.electron_temperature.loc[dict()] = te_fit.interp(rho=self.plasma.rho) - - + def map_profiles_to_midplane(self, blobs): + kinetic_profiles = [ + "electron_density", + "impurity_density", + "electron_temperature", + "ion_temperature", + "ion_density", + "fast_density", + "neutral_density", + "zeff" + ] + nchan = len(self.plasma.R_midplane) + chan = np.arange(nchan) + R = xr.DataArray(self.plasma.R_midplane, coords=[("channel", chan)]) + z = xr.DataArray(self.plasma.z_midplane, coords=[("channel", chan)]) + + rho = ( + self.plasma.equilibrium.rho + .interp(t=self.plasma.t, R=R, z=z) + .drop_vars(["R", "z"]) + ) + midplane_profiles = {} + for profile in kinetic_profiles: + midplane_profiles[profile] = blobs[profile].interp(rho_poloidal=rho).drop_vars("rho_poloidal") + midplane_profiles[profile]["R"] = R + midplane_profiles[profile]["z"] = z + midplane_profiles[profile] = midplane_profiles[profile].swap_dims({"channel": "R"}) + return midplane_profiles @dataclass class ModelSettings: @@ -926,12 +952,14 @@ def __call__( results.append(self.optimiser_context.format_results()) self.optimiser_context.optimiser.reset() + # unpack results and add time axis blobs = {} for key in results[0]["blobs"].keys(): _blob = [result["blobs"][key] for result in results] blobs[key] = xr.concat(_blob, self.plasma_context.plasma.t) self.blobs = blobs + self.midplane_blobs = self.plasma_context.map_profiles_to_midplane(blobs) opt_samples = {} for key in results[0].keys(): @@ -965,14 +993,14 @@ def __call__( if __name__ == "__main__": - pulse = None + pulse = 11089 tstart = 0.05 - tend = 0.06 + tend = 0.12 dt = 0.01 diagnostics = [ "xrcs", - # "efit", + "efit", # "smmh1", # "cxff_pi", "cxff_tws_c", @@ -987,19 +1015,19 @@ def __call__( "ts.ne", ] opt_params = [ - "Ne_prof.y0", - "Te_prof.y0", + # "Ne_prof.y0", + # "Te_prof.y0", # "Te_prof.peaking", # "Te_prof.wped", # "Te_prof.wcenter", "Ti_prof.y0", - # "Ti_prof.peaking", - # "Ti_prof.wped", - # "Ti_prof.wcenter", - # "Nimp_prof.y0", - # "Nimp_prof.peaking", - # "Nimp_prof.wcenter", - # "Nimp_prof.wped", + "Ti_prof.peaking", + "Ti_prof.wped", + "Ti_prof.wcenter", + "Nimp_prof.y0", + "Nimp_prof.peaking", + "Nimp_prof.wcenter", + "Nimp_prof.wped", # "Ne_prof.y0", ] @@ -1008,13 +1036,13 @@ def __call__( opt_quantity=opt_quant, priors=DEFAULT_PRIORS, ) data_settings = ReaderSettings(filters={}, - revisions={}) # Add general methods for filtering data co-ords to ReadST40 - data_context = MockData(pulse=pulse, diagnostics=diagnostics, - tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + revisions={"efit": "RUN02"}) # Add general methods for filtering data co-ords to ReadST40 + # data_context = MockData(pulse=pulse, diagnostics=diagnostics, + # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) # data_context = PhantomData(pulse=pulse, diagnostics=diagnostics, # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) - # data_context = ExpData(pulse=pulse, diagnostics=diagnostics, - # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + data_context = ExpData(pulse=pulse, diagnostics=diagnostics, + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context.read_data() plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), @@ -1034,7 +1062,7 @@ def __call__( plasma_context.init_plasma(equilibrium=data_context.equilibrium, tstart=tstart, tend=tend, dt=dt, ) - # plasma_context.set_ts_profiles(pulse, tstart, tend, dt, split="LFS") + plasma_context.set_ts_profiles(data_context, split="LFS") plasma_context.save_phantom_profiles(phantoms=data_context.phantoms) @@ -1043,9 +1071,9 @@ def __call__( data_context.process_data(model_context._build_bckc, ) - optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=1000, + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=2000, sample_method="high_density", starting_samples=100, burn_frac=0.20, - stopping_criteria="mode", stopping_criteria_factor=0.01, stopping_criteria_debug=True, + stopping_criteria="mode", stopping_criteria_factor=0.002, stopping_criteria_debug=True, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) @@ -1054,4 +1082,4 @@ def __call__( optimiser_context=optimiser_context, plasma_context=plasma_context, model_context=model_context) - workflow(pulse_to_write=43000000, run="RUN01", mds_write=True, plot=True, filepath=f"./results/test/") + workflow(pulse_to_write=43011089, run="RUN02", mds_write=True, plot=True, filepath=f"./results/11089_EFIT_2/") From 2b9555a40ac389318da5bf6fbfcd05317900cc7b Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 12:06:23 +0000 Subject: [PATCH 263/288] violin plots take error as input --- indica/workflows/bayes_plots.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index b70cb92d..dfe31fed 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -149,6 +149,7 @@ def _plot_1d( def violinplot( data, diag_data, + diag_data_err, filename: str, xlabel="", ylabel="[a.u.]", @@ -163,10 +164,6 @@ def violinplot( _data = data[ ((data > np.quantile(data, 0.16)) & (data < np.quantile(data, 0.84))) ] - if hasattr(diag_data, "error"): - diag_data_err = diag_data.error - else: - diag_data_err = diag_data * 0.10 violin = axs.violinplot( _data, @@ -269,6 +266,16 @@ def plot_bayes_result( param_names = results["INPUT"]["PARAM_NAMES"] phantom_profiles = flatdict.FlatDict(results["PHANTOMS"], ".") + diag_data_err = {} + for key, value in diag_data.items(): + if hasattr(value, "error"): + error = value.error + if np.any(error == 0): + error = value * 0.10 + else: + error = value * 0.10 + diag_data_err[key] = error + # select time index for plotting for t_idx, t in enumerate(time): figheader = filepath+f"t:{t.values:.2f}/" @@ -283,6 +290,7 @@ def plot_bayes_result( violinplot( model_data[key].sel(t=t), diag_data[key].sel(t=t), + diag_data_err[key].sel(t=t), f"{key.replace('.', '_')}" + filetype, xlabel=key, figheader=figheader, @@ -293,6 +301,7 @@ def plot_bayes_result( violinplot( model_data[key].sel(t=t), diag_data[key].sel(t=t), + diag_data_err[key].sel(t=t), f"{key.replace('.', '_')}" + filetype, figheader=figheader, xlabel=key, @@ -303,6 +312,7 @@ def plot_bayes_result( violinplot( model_data[key].sel(t=t), diag_data[key].sel(t=t), + diag_data_err[key].sel(t=t), f"{key.replace('.', '_')}" + filetype, figheader=figheader, xlabel=key, @@ -313,6 +323,7 @@ def plot_bayes_result( violinplot( model_data[key].sel(t=t), diag_data[key].sel(t=t), + diag_data_err[key].sel(t=t), f"{key.replace('.', '_')}" + filetype, figheader=figheader, xlabel = key, From e5a404b33906c5aeabb3357666dbcf64f1b94cb4 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 12:06:45 +0000 Subject: [PATCH 264/288] example workflow corrected --- indica/workflows/bayes_workflow.py | 52 ++++++++++++++++-------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 8c3ce6d1..eae6ed2b 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -698,7 +698,9 @@ class MockData(PhantomData): "smmh1": smmh1_transform_example(1), "cxff_pi": pi_transform_example(5), "cxff_tws_c": pi_transform_example(3), - "ts": ts_transform_example(11), }) + "ts": ts_transform_example(11), + "efit": lambda: None, # placeholder to stop missing_transforms error + }) def read_data(self): print("Reading mock equilibrium / transforms") @@ -993,29 +995,32 @@ def __call__( if __name__ == "__main__": - pulse = 11089 + pulse = None tstart = 0.05 - tend = 0.12 + tend = 0.06 dt = 0.01 diagnostics = [ "xrcs", - "efit", + # "efit", # "smmh1", - # "cxff_pi", + "cxff_pi", "cxff_tws_c", - "ts", + # "ts", ] # diagnostic_quantities opt_quant = [ "xrcs.spectra", + # "efit.wp", + # "smmh1.ne", + "cxff_pi.ti", "cxff_tws_c.ti", - # "efit.wp", - "ts.te", - "ts.ne", + # "ts.te", + # "ts.ne", ] opt_params = [ # "Ne_prof.y0", + # "Ne_prof.peaking", # "Te_prof.y0", # "Te_prof.peaking", # "Te_prof.wped", @@ -1023,12 +1028,11 @@ def __call__( "Ti_prof.y0", "Ti_prof.peaking", "Ti_prof.wped", - "Ti_prof.wcenter", - "Nimp_prof.y0", - "Nimp_prof.peaking", - "Nimp_prof.wcenter", - "Nimp_prof.wped", - # "Ne_prof.y0", + # "Ti_prof.wcenter", + # "Nimp_prof.y0", + # "Nimp_prof.peaking", + # "Nimp_prof.wcenter", + # "Nimp_prof.wped", ] # BlackBoxSettings @@ -1036,13 +1040,13 @@ def __call__( opt_quantity=opt_quant, priors=DEFAULT_PRIORS, ) data_settings = ReaderSettings(filters={}, - revisions={"efit": "RUN02"}) # Add general methods for filtering data co-ords to ReadST40 - # data_context = MockData(pulse=pulse, diagnostics=diagnostics, - # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + revisions={}) # Add general methods for filtering data co-ords to ReadST40 + data_context = MockData(pulse=pulse, diagnostics=diagnostics, + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) # data_context = PhantomData(pulse=pulse, diagnostics=diagnostics, # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) - data_context = ExpData(pulse=pulse, diagnostics=diagnostics, - tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + # data_context = ExpData(pulse=pulse, diagnostics=diagnostics, + # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) data_context.read_data() plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), impurity_concentration=(0.001, 0.04), @@ -1062,7 +1066,7 @@ def __call__( plasma_context.init_plasma(equilibrium=data_context.equilibrium, tstart=tstart, tend=tend, dt=dt, ) - plasma_context.set_ts_profiles(data_context, split="LFS") + # plasma_context.set_ts_profiles(data_context, split="LFS") plasma_context.save_phantom_profiles(phantoms=data_context.phantoms) @@ -1071,8 +1075,8 @@ def __call__( data_context.process_data(model_context._build_bckc, ) - optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=2000, - sample_method="high_density", starting_samples=100, burn_frac=0.20, + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=1000, + sample_method="high_density", starting_samples=50, burn_frac=0.20, stopping_criteria="mode", stopping_criteria_factor=0.002, stopping_criteria_debug=True, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) @@ -1082,4 +1086,4 @@ def __call__( optimiser_context=optimiser_context, plasma_context=plasma_context, model_context=model_context) - workflow(pulse_to_write=43011089, run="RUN02", mds_write=True, plot=True, filepath=f"./results/11089_EFIT_2/") + workflow(pulse_to_write=43000000, run="RUN01", mds_write=True, plot=True, filepath=f"./results/test/") From 6be843f68dac2853ad697cf98b7a464c3a8de194 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 15:28:14 +0000 Subject: [PATCH 265/288] error handling for empty data dicts --- indica/workflows/bayes_plots.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/indica/workflows/bayes_plots.py b/indica/workflows/bayes_plots.py index dfe31fed..2d1a54e5 100644 --- a/indica/workflows/bayes_plots.py +++ b/indica/workflows/bayes_plots.py @@ -272,6 +272,8 @@ def plot_bayes_result( error = value.error if np.any(error == 0): error = value * 0.10 + elif isinstance(value, flatdict.FlatDict): + continue else: error = value * 0.10 diag_data_err[key] = error From f9e9452876ff6fa0c0a58da684fae3bf40133fe6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 15:44:04 +0000 Subject: [PATCH 266/288] updated to include set_ts option --- indica/workflows/bda_run.py | 235 ++++++++---------------------------- 1 file changed, 48 insertions(+), 187 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index 42605fb0..218a2097 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -24,6 +24,27 @@ "Ti_prof.peaking", ] +PARAMS_SET_TS = [ + # "Ne_prof.y1", + # "Ne_prof.y0", + # "Ne_prof.peaking", + # "Ne_prof.wcenter", + # "Ne_prof.wped", + # "Nimp_prof.y1", + "Nimp_prof.y0", + # "Nimp_prof.wcenter", + # "Nimp_prof.wped", + "Nimp_prof.peaking", + # "Te_prof.y0", + # "Te_prof.wped", + # "Te_prof.wcenter", + # "Te_prof.peaking", + "Ti_prof.y0", + # "Ti_prof.wped", + "Ti_prof.wcenter", + "Ti_prof.peaking", +] + DIAG_DEFAULT = [ "xrcs", "ts", @@ -63,9 +84,9 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, - tstart=0.02, tend=0.03, dt=0.01, revisions = {}, filters={}, - iterations=500, nwalkers=50, stopping_criteria_factor=0.01, - mds_write=False, plot=False, run="RUN01", dirname=None, **kwargs): + tstart=0.02, tend=0.05, dt=0.01, revisions = {}, filters={}, + iterations=500, nwalkers=50, stopping_criteria_factor=0.002, + mds_write=False, plot=False, run="RUN01", dirname=None, set_ts=False, **kwargs): if dirname is None: dirname = f"{pulse}" @@ -88,7 +109,13 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, n_rad=20) plasma_context = PlasmaContext(plasma_settings=plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) - model_settings = ModelSettings() + plasma_context.init_plasma(data_context.equilibrium, tstart=tstart, tend=tend, dt=dt) + plasma_context.save_phantom_profiles(phantoms=data_context.phantoms) + + if set_ts: + plasma_context.set_ts_profiles(data_context, split="LFS") + + model_settings = ModelSettings(call_kwargs={"xrcs": {"pixel_offset": 0.0}}) model_context = ModelContext(diagnostics=diagnostics, plasma_context=plasma_context, @@ -96,9 +123,13 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, transforms=data_context.transforms, model_settings=model_settings, ) + model_context.update_model_kwargs(data_context.binned_data) + model_context.init_models() + + data_context.process_data(model_context._build_bckc, ) optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=nwalkers, iterations=iterations, - sample_method="high_density", starting_samples=300, burn_frac=0.20, + sample_method="high_density", starting_samples=100, burn_frac=0.20, stopping_criteria="mode", stopping_criteria_factor=stopping_criteria_factor, stopping_criteria_debug=True, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) @@ -108,85 +139,12 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, optimiser_context=optimiser_context, plasma_context=plasma_context, model_context=model_context) - workflow(pulse_to_write=25000000+pulse, run=run, mds_write=mds_write, plot=plot, filepath=f"./results/{dirname}/") + workflow(pulse_to_write=43000000+pulse, run=run, mds_write=mds_write, plot=plot, filepath=f"./results/{dirname}/") if __name__ == "__main__": pulse_info = [ - # stopping criteria test / integration test - # [(11226, - # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - # PARAMS_DEFAULT, - # [ - # "xrcs.spectra", - # "cxff_pi.ti", - # "cxff_tws_c.ti", - # "ts.ne", - # "ts.te", - # ]), - # dict( - # phantom=True, - # tstart=0.07, - # tend=0.08, - # dt=0.01, - # iterations=5000, - # nwalkers=26, - # stopping_criteria_factor=0.01, - # mds_write=True, - # plot=True, - # run="RUN01", - # dirname=f"{11226}_phantom_walkers_26", - # )], - - # [(11226, - # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - # PARAMS_DEFAULT, - # [ - # "xrcs.spectra", - # "cxff_pi.ti", - # "cxff_tws_c.ti", - # "ts.ne", - # "ts.te", - # ]), - # dict( - # phantom=True, - # tstart=0.07, - # tend=0.08, - # dt=0.01, - # iterations=5000, - # nwalkers=50, - # stopping_criteria_factor=0.01, - # mds_write=True, - # plot=True, - # run="RUN01", - # dirname=f"{11226}_phantom_walkers_50", - # )], - # - # [(11226, - # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - # PARAMS_DEFAULT, - # [ - # "xrcs.spectra", - # "cxff_pi.ti", - # "cxff_tws_c.ti", - # "ts.ne", - # "ts.te", - # ]), - # dict( - # phantom=True, - # tstart=0.07, - # tend=0.08, - # dt=0.01, - # iterations=5000, - # nwalkers=100, - # stopping_criteria_factor=0.01, - # mds_write=True, - # plot=True, - # run="RUN01", - # dirname=f"{11226}_phantom_walkers_100", - # )], - # [(11226, # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], @@ -200,92 +158,18 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # ]), # dict( # phantom=True, - # tstart=0.07, - # tend=0.08, - # dt=0.01, - # iterations=5000, - # nwalkers=50, - # stopping_criteria_factor=0.02, - # mds_write=True, - # plot=True, - # run="RUN01", - # dirname=f"{11226}_phantom_moment_02", - # )], - # - # [(11226, - # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - # PARAMS_DEFAULT, - # [ - # "xrcs.spectra", - # "cxff_pi.ti", - # "cxff_tws_c.ti", - # "ts.ne", - # "ts.te", - # ]), - # dict( - # phantom=True, - # tstart=0.07, - # tend=0.08, - # dt=0.01, - # iterations=5000, - # nwalkers=50, - # stopping_criteria_factor=0.01, - # mds_write=True, - # plot=True, - # run="RUN01", - # dirname=f"{11226}_phantom_moment_01", - # )], - # - # [(11226, - # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - # PARAMS_DEFAULT, - # [ - # "xrcs.spectra", - # "cxff_pi.ti", - # "cxff_tws_c.ti", - # "ts.ne", - # "ts.te", - # ]), - # dict( - # phantom=True, - # tstart=0.07, - # tend=0.08, - # dt=0.01, - # iterations=5000, - # nwalkers=50, - # stopping_criteria_factor=0.001, - # mds_write=True, - # plot=True, - # run="RUN01", - # dirname=f"{11226}_phantom_moment_001", - # )], - - # [(11226, - # ["xrcs", "cxff_pi", "cxff_tws_c", "ts"], - # PARAMS_DEFAULT, - # [ - # "xrcs.spectra", - # "cxff_pi.ti", - # "cxff_tws_c.ti", - # "ts.ne", - # "ts.te", - # ]), - # dict( - # phantom=False, - # tstart=0.07, - # tend=0.08, + # tstart=0.05, + # tend=0.10, # dt=0.01, # iterations=5000, - # stopping_criteria_factor=0.01, # nwalkers=50, + # stopping_criteria_factor=0.002, # mds_write=True, # plot=True, # run="RUN01", - # dirname=f"{11226}_exp", + # dirname=f"{11226}", # )], - - # [(10009, # ["xrcs", "cxff_pi"], # PARAMS_DEFAULT, @@ -306,33 +190,9 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # dirname=f"{11336}_phantom", # )], - # [(11089, - # ["xrcs", "cxff_tws_c", "ts"], - # PARAMS_DEFAULT, - # [ - # "xrcs.spectra", - # "cxff_tws_c.ti", - # "ts.ne", - # "ts.te", - # ]), - # dict( - # phantom=False, - # tstart=0.050, - # tend=0.150, - # dt=0.01, - # iterations=5000, - # nwalkers=50, - # stopping_criteria_factor=0.01, - # mds_write=True, - # plot=True, - # revisions={}, - # run="RUN01", - # dirname=f"{11089}_quicktest", - # )], - [(11089, - ["xrcs", "cxff_tws_c", "ts"], - PARAMS_DEFAULT, + ["xrcs", "cxff_tws_c", "ts", "efit"], + PARAMS_SET_TS, [ "xrcs.spectra", "cxff_tws_c.ti", @@ -342,16 +202,17 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, dict( phantom=False, tstart=0.05, - tend=0.12, + tend=0.10, dt=0.01, - iterations=5000, - nwalkers=50, + iterations=1000, + nwalkers=20, stopping_criteria_factor=0.002, mds_write=True, plot=True, revisions={}, run="RUN01", - dirname=f"{11089}_DEWALKER_NOSNOOKER", + # dirname="test", + set_ts=True, )], # RFX low ti/te pulse ] From a14650ede238ac218788da204f21afe9c2d666b7 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 17:05:38 +0000 Subject: [PATCH 267/288] python 3.8 -> poetry 3.9 --- poetry.lock | 1520 +++++++++++++++++++++++++++------------------------ 1 file changed, 809 insertions(+), 711 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5d84cb92..e7d0ab06 100644 --- a/poetry.lock +++ b/poetry.lock @@ -19,17 +19,18 @@ python-versions = "*" [[package]] name = "asttokens" -version = "2.2.1" +version = "2.4.1" description = "Annotate AST trees with source code positions" category = "main" optional = false python-versions = "*" [package.dependencies] -six = "*" +six = ">=1.12.0" [package.extras] -test = ["astroid", "pytest"] +astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"] +test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] [[package]] name = "atomicwrites" @@ -41,7 +42,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" category = "dev" optional = false @@ -52,7 +53,8 @@ cov = ["attrs", "coverage[toml] (>=5.3)"] dev = ["attrs", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist"] [[package]] name = "backcall" @@ -64,15 +66,15 @@ python-versions = "*" [[package]] name = "blinker" -version = "1.6.2" +version = "1.7.0" description = "Fast, simple object-to-object and broadcast signaling" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [[package]] name = "certifi" -version = "2023.5.7" +version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -80,26 +82,26 @@ python-versions = ">=3.6" [[package]] name = "cfgv" -version = "3.3.1" +version = "3.4.0" description = "Validate configuration and produce human readable error messages." category = "dev" optional = false -python-versions = ">=3.6.1" +python-versions = ">=3.8" [[package]] name = "cftime" -version = "1.6.2" +version = "1.6.3" description = "Time-handling functionality from netcdf4-python" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.dependencies] -numpy = ">1.13.3" +numpy = {version = ">1.13.3", markers = "python_version < \"3.12.0.rc1\""} [[package]] name = "charset-normalizer" -version = "3.1.0" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false @@ -107,7 +109,7 @@ python-versions = ">=3.7.0" [[package]] name = "click" -version = "8.1.3" +version = "8.1.7" description = "Composable command line interface toolkit" category = "main" optional = false @@ -126,19 +128,19 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7 [[package]] name = "contourpy" -version = "1.1.0" +version = "1.1.1" description = "Python library for calculating contours of 2D quadrilateral grids" category = "main" optional = false python-versions = ">=3.8" [package.dependencies] -numpy = ">=1.16" +numpy = {version = ">=1.16,<2.0", markers = "python_version <= \"3.11\""} [package.extras] bokeh = ["bokeh", "selenium"] -docs = ["furo", "sphinx-copybutton"] -mypy = ["contourpy", "docutils-stubs", "mypy (==1.2.0)", "types-pillow"] +docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] +mypy = ["contourpy", "docutils-stubs", "mypy (==1.4.1)", "types-pillow"] test = ["contourpy", "matplotlib", "pillow"] test-no-images = ["pytest", "pytest-cov", "wurlitzer"] @@ -166,11 +168,15 @@ toml = ["toml"] [[package]] name = "cycler" -version = "0.11.0" +version = "0.12.1" description = "Composable style cycles" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" + +[package.extras] +docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] +tests = ["pytest", "pytest-cov", "pytest-xdist"] [[package]] name = "decorator" @@ -182,7 +188,7 @@ python-versions = ">=3.5" [[package]] name = "distlib" -version = "0.3.6" +version = "0.3.8" description = "Distribution utilities" category = "dev" optional = false @@ -205,41 +211,42 @@ tests = ["coverage", "pytest", "pytest-cov"] [[package]] name = "execnet" -version = "1.9.0" +version = "2.0.2" description = "execnet: rapid multi-Python deployment" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7" [package.extras] -testing = ["pre-commit"] +testing = ["hatch", "pre-commit", "pytest", "tox"] [[package]] name = "executing" -version = "1.2.0" +version = "2.0.1" description = "Get the currently executing AST node of a frame, and other information" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.5" [package.extras] -tests = ["asttokens", "littleutils", "pytest", "rich"] +tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] [[package]] name = "filelock" -version = "3.12.2" +version = "3.13.1" description = "A platform independent file lock." category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.extras] -docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] [[package]] name = "flask" -version = "2.3.2" +version = "3.0.0" description = "A simple framework for building complex web applications." category = "main" optional = false @@ -251,7 +258,7 @@ click = ">=8.1.3" importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""} itsdangerous = ">=2.1.2" Jinja2 = ">=3.1.2" -Werkzeug = ">=2.3.3" +Werkzeug = ">=3.0.0" [package.extras] async = ["asgiref (>=3.2)"] @@ -284,16 +291,16 @@ python-versions = "*" [[package]] name = "fonttools" -version = "4.40.0" +version = "4.47.2" description = "Tools to manipulate font files" category = "main" optional = false python-versions = ">=3.8" [package.extras] -all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "fs (>=2.2.0,<3)", "lxml (>=4.0,<5)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.0.0)", "xattr", "zopfli (>=0.1.4)"] +all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "fs (>=2.2.0,<3)", "lxml (>=4.0,<5)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "pycairo", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0)", "xattr", "zopfli (>=0.1.4)"] graphite = ["lz4 (>=1.7.4.2)"] -interpolatable = ["munkres", "scipy"] +interpolatable = ["munkres", "pycairo", "scipy"] lxml = ["lxml (>=4.0,<5)"] pathops = ["skia-pathops (>=0.5.0)"] plot = ["matplotlib"] @@ -301,7 +308,7 @@ repacker = ["uharfbuzz (>=0.23.0)"] symfont = ["sympy"] type1 = ["xattr"] ufo = ["fs (>=2.2.0,<3)"] -unicode = ["unicodedata2 (>=15.0.0)"] +unicode = ["unicodedata2 (>=15.1.0)"] woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] [[package]] @@ -335,18 +342,18 @@ zoneinfo = ["backports.zoneinfo (>=0.2.1)", "importlib-resources (>=3.3.0)", "tz [[package]] name = "identify" -version = "2.5.24" +version = "2.5.33" description = "File identification library for Python" category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.extras] license = ["ukkonen"] [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false @@ -354,34 +361,34 @@ python-versions = ">=3.5" [[package]] name = "importlib-metadata" -version = "6.7.0" +version = "7.0.1" description = "Read metadata from Python packages" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] [[package]] name = "importlib-resources" -version = "5.12.0" +version = "6.1.1" description = "Read resources from Python packages" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.dependencies] zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff", "zipp (>=3.17)"] [[package]] name = "iniconfig" @@ -448,23 +455,23 @@ python-versions = ">=3.7" [[package]] name = "jedi" -version = "0.18.2" +version = "0.19.1" description = "An autocompletion tool for Python that can be used for text editors." category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -parso = ">=0.8.0,<0.9.0" +parso = ">=0.8.3,<0.9.0" [package.extras] docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] -qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] -testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] +testing = ["attrs", "colorama", "django", "docopt", "pytest (<7.0.0)"] [[package]] name = "jinja2" -version = "3.1.2" +version = "3.1.3" description = "A very fast and expressive template engine." category = "main" optional = false @@ -478,7 +485,7 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "joblib" -version = "1.3.1" +version = "1.3.2" description = "Lightweight pipelining with Python functions" category = "main" optional = false @@ -486,7 +493,7 @@ python-versions = ">=3.7" [[package]] name = "kiwisolver" -version = "1.4.4" +version = "1.4.5" description = "A fast implementation of the Cassowary constraint solver" category = "main" optional = false @@ -494,17 +501,17 @@ python-versions = ">=3.7" [[package]] name = "lxml" -version = "4.9.2" +version = "5.1.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" +python-versions = ">=3.6" [package.extras] cssselect = ["cssselect (>=0.7)"] html5 = ["html5lib"] htmlsoup = ["beautifulsoup4"] -source = ["Cython (>=0.29.7)"] +source = ["Cython (>=3.0.7)"] [[package]] name = "markupsafe" @@ -516,7 +523,7 @@ python-versions = ">=3.7" [[package]] name = "matplotlib" -version = "3.7.1" +version = "3.7.4" description = "Python plotting package" category = "main" optional = false @@ -528,12 +535,11 @@ cycler = ">=0.10" fonttools = ">=4.22.0" importlib-resources = {version = ">=3.2.0", markers = "python_version < \"3.10\""} kiwisolver = ">=1.0.1" -numpy = ">=1.20" +numpy = ">=1.20,<2" packaging = ">=20.0" pillow = ">=6.2.0" pyparsing = ">=2.3.1" python-dateutil = ">=2.7" -setuptools_scm = ">=7" [[package]] name = "matplotlib-inline" @@ -574,7 +580,7 @@ python-versions = ">=3.5" [[package]] name = "netcdf4" -version = "1.6.4" +version = "1.6.5" description = "Provides an object-oriented python interface to the netCDF version 4 library" category = "main" optional = false @@ -585,6 +591,9 @@ certifi = "*" cftime = "*" numpy = "*" +[package.extras] +tests = ["cython", "packaging", "pytest"] + [[package]] name = "networkx" version = "3.1" @@ -618,7 +627,7 @@ python-versions = ">=3.7" [[package]] name = "packaging" -version = "23.1" +version = "23.2" description = "Core utilities for Python packages" category = "main" optional = false @@ -654,7 +663,7 @@ testing = ["docopt", "pytest (<6.0.0)"] [[package]] name = "pexpect" -version = "4.8.0" +version = "4.9.0" description = "Pexpect allows easy control of interactive console applications." category = "main" optional = false @@ -673,35 +682,39 @@ python-versions = "*" [[package]] name = "pillow" -version = "9.5.0" +version = "10.2.0" description = "Python Imaging Library (Fork)" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.extras] docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] [[package]] name = "platformdirs" -version = "3.8.0" +version = "4.1.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] [[package]] name = "pluggy" -version = "1.2.0" +version = "1.3.0" description = "plugin and hook calling mechanisms for python" category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.extras] dev = ["pre-commit", "tox"] @@ -724,7 +737,7 @@ virtualenv = ">=20.10.0" [[package]] name = "prompt-toolkit" -version = "3.0.38" +version = "3.0.43" description = "Library for building powerful interactive command lines in Python" category = "main" optional = false @@ -780,7 +793,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pygments" -version = "2.15.1" +version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false @@ -788,10 +801,11 @@ python-versions = ">=3.7" [package.extras] plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyparsing" -version = "3.1.0" +version = "3.1.1" description = "pyparsing module - Classes and methods to define and execute parsing grammars" category = "main" optional = false @@ -880,7 +894,7 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2023.3" +version = "2023.3.post1" description = "World timezone definitions, modern and historical" category = "main" optional = false @@ -888,7 +902,7 @@ python-versions = "*" [[package]] name = "pyyaml" -version = "6.0" +version = "6.0.1" description = "YAML parser and emitter for Python" category = "dev" optional = false @@ -950,7 +964,7 @@ ldap = ["ldap3"] [[package]] name = "scikit-learn" -version = "1.3.0" +version = "1.3.2" description = "A set of python modules for machine learning and data mining" category = "main" optional = false @@ -958,7 +972,7 @@ python-versions = ">=3.8" [package.dependencies] joblib = ">=1.1.1" -numpy = ">=1.17.3" +numpy = ">=1.17.3,<2.0" scipy = ">=1.5.0" threadpoolctl = ">=2.0.0" @@ -984,23 +998,6 @@ dev = ["click", "doit (>=0.36.0)", "flake8", "mypy", "pycodestyle", "pydevtool", doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-design (>=0.2.0)"] test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] -[[package]] -name = "setuptools-scm" -version = "7.1.0" -description = "the blessed package to manage your versions by scm tags" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -packaging = ">=20.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} -typing-extensions = "*" - -[package.extras] -test = ["pytest (>=6.2)", "virtualenv (>20)"] -toml = ["setuptools (>=42)"] - [[package]] name = "six" version = "1.16.0" @@ -1019,7 +1016,7 @@ python-versions = "*" [[package]] name = "stack-data" -version = "0.6.2" +version = "0.6.3" description = "Extract data from python stack frames and tracebacks for informative displays" category = "main" optional = false @@ -1035,11 +1032,11 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] [[package]] name = "threadpoolctl" -version = "3.1.0" +version = "3.2.0" description = "threadpoolctl" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" [[package]] name = "toml" @@ -1053,13 +1050,13 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "main" +category = "dev" optional = false python-versions = ">=3.7" [[package]] name = "tqdm" -version = "4.65.0" +version = "4.66.1" description = "Fast, Extensible Progress Meter" category = "main" optional = false @@ -1069,22 +1066,22 @@ python-versions = ">=3.7" colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] [[package]] name = "traitlets" -version = "5.9.0" +version = "5.14.1" description = "Traitlets Python configuration system" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.extras] docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] -test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] +test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<7.5)", "pytest-mock", "pytest-mypy-testing"] [[package]] name = "types-setuptools" @@ -1096,46 +1093,45 @@ python-versions = "*" [[package]] name = "typing-extensions" -version = "4.7.0" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [[package]] name = "urllib3" -version = "2.0.3" +version = "2.1.0" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" -version = "20.23.1" +version = "20.25.0" description = "Virtual Python Environment builder" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.12,<4" -platformdirs = ">=3.5.1,<4" +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<5" [package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.3.1)", "pytest-env (>=0.8.1)", "pytest-freezer (>=0.4.6)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=67.8)", "time-machine (>=2.9)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] [[package]] name = "wcwidth" -version = "0.2.6" +version = "0.2.13" description = "Measures the displayed width of unicode strings in a terminal" category = "main" optional = false @@ -1143,7 +1139,7 @@ python-versions = "*" [[package]] name = "werkzeug" -version = "2.3.6" +version = "3.0.1" description = "The comprehensive WSGI web application library." category = "main" optional = false @@ -1177,15 +1173,15 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [[package]] name = "zipp" -version = "3.15.0" +version = "3.17.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-o", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-o", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] [metadata] lock-version = "1.1" @@ -1202,180 +1198,208 @@ appnope = [ {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, ] asttokens = [ - {file = "asttokens-2.2.1-py2.py3-none-any.whl", hash = "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"}, - {file = "asttokens-2.2.1.tar.gz", hash = "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3"}, + {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, + {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, ] atomicwrites = [ {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, ] attrs = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] backcall = [ {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, ] blinker = [ - {file = "blinker-1.6.2-py3-none-any.whl", hash = "sha256:c3d739772abb7bc2860abf5f2ec284223d9ad5c76da018234f6f50d6f31ab1f0"}, - {file = "blinker-1.6.2.tar.gz", hash = "sha256:4afd3de66ef3a9f8067559fb7a1cbe555c17dcbe15971b05d1b625c3e7abe213"}, + {file = "blinker-1.7.0-py3-none-any.whl", hash = "sha256:c3f865d4d54db7abc53758a01601cf343fe55b84c1de4e3fa910e420b438d5b9"}, + {file = "blinker-1.7.0.tar.gz", hash = "sha256:e6820ff6fa4e4d1d8e2747c2283749c3f547e4fee112b98555cdcdae32996182"}, ] certifi = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] cfgv = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, ] cftime = [ - {file = "cftime-1.6.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b4d2a1920f0aad663f25700b30621ff64af373499e52b544da1148dd8c09409a"}, - {file = "cftime-1.6.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ba7909a0cd4adcb16797d8d6ab2767e7ddb980b2bf9dbabfc71b3bdd94f072b"}, - {file = "cftime-1.6.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acb294fdb80e33545ae54b4421df35c4e578708a5ffce1c00408b2294e70ecef"}, - {file = "cftime-1.6.2-cp310-cp310-win_amd64.whl", hash = "sha256:2abdac6ca5b8b6102f319122546739dfc42406b816c16f2a98a8f0cd406d3bf0"}, - {file = "cftime-1.6.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:eb7f8cd0996640b83020133b5ef6b97fc9216c3129eaeeaca361abdff5d82166"}, - {file = "cftime-1.6.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d49d69c64cee2c175478eed84c3a57fce083da4ceebce16440f72be561a8489"}, - {file = "cftime-1.6.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:455cec3627e6ca8694b0d9201da6581eb4381b58389f1fbcb51a14fa0e2b3d94"}, - {file = "cftime-1.6.2-cp311-cp311-win_amd64.whl", hash = "sha256:29c18601abea0fd160fbe423e05c7a56fe1d38dd250a6b010de499a132d3fe18"}, - {file = "cftime-1.6.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:afb5b38b51b8bc02f1656a9f15c52b0b20a3999adbe1ab9ac57f926e0065b48a"}, - {file = "cftime-1.6.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aedfb7a783d19d7a30cb41951310f3bfe98f9f21fffc723c8af08a11962b0b17"}, - {file = "cftime-1.6.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3042048324b4d6a1066c978ec78101effdd84320e8862bfdbf8122d7ad7588ec"}, - {file = "cftime-1.6.2-cp37-none-win_amd64.whl", hash = "sha256:ee70fa069802652cf534de1dd3fc590b7d22d4127447bf96ac9849abcdadadf1"}, - {file = "cftime-1.6.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:93f00f454329c1f2588ebca2650e8edf7607d6189dbdcc81b5f3be2080155cc4"}, - {file = "cftime-1.6.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e83db2fdda900eb154a9f79dfb665ac6190781c61d2e18151996de5ee7ffd8a2"}, - {file = "cftime-1.6.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56d0242fc4990584b265622622b25bb262a178097711d2d95e53ef52a9d23e7e"}, - {file = "cftime-1.6.2-cp38-cp38-win_amd64.whl", hash = "sha256:055d5d60a756c6c1857cf84d77655bb707057bb6c4a4fbb104a550e76c40aad9"}, - {file = "cftime-1.6.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0955e1f3e1c09a9e0296b50f135ff9719cb2466f81c8ad4a10ef06fa394de984"}, - {file = "cftime-1.6.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:07fdef2f75a0f0952b0376fa4cd08ef8a1dad3b963976ac07517811d434936b7"}, - {file = "cftime-1.6.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:892d5dc38f8b998c83a2a01f131e63896d020586de473e1878f9e85acc70ad44"}, - {file = "cftime-1.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:86fe550b94525c327578a90b2e13418ca5ba6c636d5efe3edec310e631757eea"}, - {file = "cftime-1.6.2.tar.gz", hash = "sha256:8614c00fb8a5046de304fdd86dbd224f99408185d7b245ac6628d0276596e6d2"}, + {file = "cftime-1.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b62d42546fa5c914dfea5b15a9aaed2087ea1211cc36d08c374502ef95892038"}, + {file = "cftime-1.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb6dd70b2ccabfe1a14b7fbb0bbdce0418e71697094373c0d573c880790fa291"}, + {file = "cftime-1.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9878bfd8c1c3f24184ecbd528f739ba46ebaceaf1c8a24d348d7befb117a285"}, + {file = "cftime-1.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:3cf6e216a4c06f9a628cdf8e9c9d5e8097fb3eb02dd087dd14ab3b18478a7271"}, + {file = "cftime-1.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d2c01456d9d7b46aa710a41d1c711a50d5ea259aff4a987d0e973d1093bc922"}, + {file = "cftime-1.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:80eb1170ce1639016f55760847f4aadd04b0312496c5bac2797e930914bba48d"}, + {file = "cftime-1.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d87dadd0824262bdd7493babd2a44447da0a22175ded8ae9e060a3aebec7c5d7"}, + {file = "cftime-1.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:0a38eb9f5c733a23e1714bd3ef2762ed5acee34f127670f8fb4ad6464946f6b3"}, + {file = "cftime-1.6.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2d113a01ab924445e61d65c26bbd95bc08e4a22878d3b947064bba056c884c4a"}, + {file = "cftime-1.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f11685663a6af97418908060492a07663c16d42519c139ca03c2ffb1377fd25"}, + {file = "cftime-1.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a98abb1d46d118e52b0611ce668a0b714b407be26177ef0581ecf5e95f894725"}, + {file = "cftime-1.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:4d6fbd5f41b322cfa7b0ac3aaadeceb4450100a164b5bccbbb9e7c5048489a88"}, + {file = "cftime-1.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bedb577bc8b8f3f10f5336c0792e5dae88605781890f50f36b45bb46907968e8"}, + {file = "cftime-1.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:022dabf1610cdd04a693e730fa8f71d307059717f29dba921e7486e553412bb4"}, + {file = "cftime-1.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbf782ab4ac0605bdec2b941952c897595613203942b7f8c2fccd17efa5147df"}, + {file = "cftime-1.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:9eb177a02db7cd84aa6962278e4bd2d3106a545de82e6aacd9404f1e153661db"}, + {file = "cftime-1.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b86be8c2f254147be4ba88f12099466dde457a4a3a21de6c69d52a7224c13ae"}, + {file = "cftime-1.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:523b9a6bf03f5e36407979e248381d0fcab2d225b915bbde77d00c6dde192b90"}, + {file = "cftime-1.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a14d2c7d22fd2a6dfa6ad563283b6d6679f1df95e0ed8d14b8f284dad402887"}, + {file = "cftime-1.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:d9b00c2844c7a1701d8ede5336b6321dfee256ceab81a34a1aff0483d56891a6"}, + {file = "cftime-1.6.3.tar.gz", hash = "sha256:d0a6b29f72a13f08e008b9becff247cc75c84acb213332ede18879c5b6aa4dfd"}, ] charset-normalizer = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] colorama = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] contourpy = [ - {file = "contourpy-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:89f06eff3ce2f4b3eb24c1055a26981bffe4e7264acd86f15b97e40530b794bc"}, - {file = "contourpy-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dffcc2ddec1782dd2f2ce1ef16f070861af4fb78c69862ce0aab801495dda6a3"}, - {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25ae46595e22f93592d39a7eac3d638cda552c3e1160255258b695f7b58e5655"}, - {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:17cfaf5ec9862bc93af1ec1f302457371c34e688fbd381f4035a06cd47324f48"}, - {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18a64814ae7bce73925131381603fff0116e2df25230dfc80d6d690aa6e20b37"}, - {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c81f22b4f572f8a2110b0b741bb64e5a6427e0a198b2cdc1fbaf85f352a3aa"}, - {file = "contourpy-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53cc3a40635abedbec7f1bde60f8c189c49e84ac180c665f2cd7c162cc454baa"}, - {file = "contourpy-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:1f795597073b09d631782e7245016a4323cf1cf0b4e06eef7ea6627e06a37ff2"}, - {file = "contourpy-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0b7b04ed0961647691cfe5d82115dd072af7ce8846d31a5fac6c142dcce8b882"}, - {file = "contourpy-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27bc79200c742f9746d7dd51a734ee326a292d77e7d94c8af6e08d1e6c15d545"}, - {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:052cc634bf903c604ef1a00a5aa093c54f81a2612faedaa43295809ffdde885e"}, - {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9382a1c0bc46230fb881c36229bfa23d8c303b889b788b939365578d762b5c18"}, - {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5cec36c5090e75a9ac9dbd0ff4a8cf7cecd60f1b6dc23a374c7d980a1cd710e"}, - {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f0cbd657e9bde94cd0e33aa7df94fb73c1ab7799378d3b3f902eb8eb2e04a3a"}, - {file = "contourpy-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:181cbace49874f4358e2929aaf7ba84006acb76694102e88dd15af861996c16e"}, - {file = "contourpy-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb3b7d9e6243bfa1efb93ccfe64ec610d85cfe5aec2c25f97fbbd2e58b531256"}, - {file = "contourpy-1.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bcb41692aa09aeb19c7c213411854402f29f6613845ad2453d30bf421fe68fed"}, - {file = "contourpy-1.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5d123a5bc63cd34c27ff9c7ac1cd978909e9c71da12e05be0231c608048bb2ae"}, - {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62013a2cf68abc80dadfd2307299bfa8f5aa0dcaec5b2954caeb5fa094171103"}, - {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0b6616375d7de55797d7a66ee7d087efe27f03d336c27cf1f32c02b8c1a5ac70"}, - {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:317267d915490d1e84577924bd61ba71bf8681a30e0d6c545f577363157e5e94"}, - {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d551f3a442655f3dcc1285723f9acd646ca5858834efeab4598d706206b09c9f"}, - {file = "contourpy-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7a117ce7df5a938fe035cad481b0189049e8d92433b4b33aa7fc609344aafa1"}, - {file = "contourpy-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4f26b25b4f86087e7d75e63212756c38546e70f2a92d2be44f80114826e1cd4"}, - {file = "contourpy-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc00bb4225d57bff7ebb634646c0ee2a1298402ec10a5fe7af79df9a51c1bfd9"}, - {file = "contourpy-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:189ceb1525eb0655ab8487a9a9c41f42a73ba52d6789754788d1883fb06b2d8a"}, - {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f2931ed4741f98f74b410b16e5213f71dcccee67518970c42f64153ea9313b9"}, - {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30f511c05fab7f12e0b1b7730ebdc2ec8deedcfb505bc27eb570ff47c51a8f15"}, - {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143dde50520a9f90e4a2703f367cf8ec96a73042b72e68fcd184e1279962eb6f"}, - {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e94bef2580e25b5fdb183bf98a2faa2adc5b638736b2c0a4da98691da641316a"}, - {file = "contourpy-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ed614aea8462735e7d70141374bd7650afd1c3f3cb0c2dbbcbe44e14331bf002"}, - {file = "contourpy-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:438ba416d02f82b692e371858143970ed2eb6337d9cdbbede0d8ad9f3d7dd17d"}, - {file = "contourpy-1.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a698c6a7a432789e587168573a864a7ea374c6be8d4f31f9d87c001d5a843493"}, - {file = "contourpy-1.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397b0ac8a12880412da3551a8cb5a187d3298a72802b45a3bd1805e204ad8439"}, - {file = "contourpy-1.1.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:a67259c2b493b00e5a4d0f7bfae51fb4b3371395e47d079a4446e9b0f4d70e76"}, - {file = "contourpy-1.1.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2b836d22bd2c7bb2700348e4521b25e077255ebb6ab68e351ab5aa91ca27e027"}, - {file = "contourpy-1.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084eaa568400cfaf7179b847ac871582199b1b44d5699198e9602ecbbb5f6104"}, - {file = "contourpy-1.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:911ff4fd53e26b019f898f32db0d4956c9d227d51338fb3b03ec72ff0084ee5f"}, - {file = "contourpy-1.1.0.tar.gz", hash = "sha256:e53046c3863828d21d531cc3b53786e6580eb1ba02477e8681009b6aa0870b21"}, + {file = "contourpy-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:46e24f5412c948d81736509377e255f6040e94216bf1a9b5ea1eaa9d29f6ec1b"}, + {file = "contourpy-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e48694d6a9c5a26ee85b10130c77a011a4fedf50a7279fa0bdaf44bafb4299d"}, + {file = "contourpy-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a66045af6cf00e19d02191ab578a50cb93b2028c3eefed999793698e9ea768ae"}, + {file = "contourpy-1.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ebf42695f75ee1a952f98ce9775c873e4971732a87334b099dde90b6af6a916"}, + {file = "contourpy-1.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6aec19457617ef468ff091669cca01fa7ea557b12b59a7908b9474bb9674cf0"}, + {file = "contourpy-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:462c59914dc6d81e0b11f37e560b8a7c2dbab6aca4f38be31519d442d6cde1a1"}, + {file = "contourpy-1.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6d0a8efc258659edc5299f9ef32d8d81de8b53b45d67bf4bfa3067f31366764d"}, + {file = "contourpy-1.1.1-cp310-cp310-win32.whl", hash = "sha256:d6ab42f223e58b7dac1bb0af32194a7b9311065583cc75ff59dcf301afd8a431"}, + {file = "contourpy-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:549174b0713d49871c6dee90a4b499d3f12f5e5f69641cd23c50a4542e2ca1eb"}, + {file = "contourpy-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:407d864db716a067cc696d61fa1ef6637fedf03606e8417fe2aeed20a061e6b2"}, + {file = "contourpy-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe80c017973e6a4c367e037cb31601044dd55e6bfacd57370674867d15a899b"}, + {file = "contourpy-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e30aaf2b8a2bac57eb7e1650df1b3a4130e8d0c66fc2f861039d507a11760e1b"}, + {file = "contourpy-1.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3de23ca4f381c3770dee6d10ead6fff524d540c0f662e763ad1530bde5112532"}, + {file = "contourpy-1.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:566f0e41df06dfef2431defcfaa155f0acfa1ca4acbf8fd80895b1e7e2ada40e"}, + {file = "contourpy-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b04c2f0adaf255bf756cf08ebef1be132d3c7a06fe6f9877d55640c5e60c72c5"}, + {file = "contourpy-1.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d0c188ae66b772d9d61d43c6030500344c13e3f73a00d1dc241da896f379bb62"}, + {file = "contourpy-1.1.1-cp311-cp311-win32.whl", hash = "sha256:0683e1ae20dc038075d92e0e0148f09ffcefab120e57f6b4c9c0f477ec171f33"}, + {file = "contourpy-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:8636cd2fc5da0fb102a2504fa2c4bea3cbc149533b345d72cdf0e7a924decc45"}, + {file = "contourpy-1.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:560f1d68a33e89c62da5da4077ba98137a5e4d3a271b29f2f195d0fba2adcb6a"}, + {file = "contourpy-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:24216552104ae8f3b34120ef84825400b16eb6133af2e27a190fdc13529f023e"}, + {file = "contourpy-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56de98a2fb23025882a18b60c7f0ea2d2d70bbbcfcf878f9067234b1c4818442"}, + {file = "contourpy-1.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:07d6f11dfaf80a84c97f1a5ba50d129d9303c5b4206f776e94037332e298dda8"}, + {file = "contourpy-1.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1eaac5257a8f8a047248d60e8f9315c6cff58f7803971170d952555ef6344a7"}, + {file = "contourpy-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19557fa407e70f20bfaba7d55b4d97b14f9480856c4fb65812e8a05fe1c6f9bf"}, + {file = "contourpy-1.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:081f3c0880712e40effc5f4c3b08feca6d064cb8cfbb372ca548105b86fd6c3d"}, + {file = "contourpy-1.1.1-cp312-cp312-win32.whl", hash = "sha256:059c3d2a94b930f4dafe8105bcdc1b21de99b30b51b5bce74c753686de858cb6"}, + {file = "contourpy-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f44d78b61740e4e8c71db1cf1fd56d9050a4747681c59ec1094750a658ceb970"}, + {file = "contourpy-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:70e5a10f8093d228bb2b552beeb318b8928b8a94763ef03b858ef3612b29395d"}, + {file = "contourpy-1.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8394e652925a18ef0091115e3cc191fef350ab6dc3cc417f06da66bf98071ae9"}, + {file = "contourpy-1.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5bd5680f844c3ff0008523a71949a3ff5e4953eb7701b28760805bc9bcff217"}, + {file = "contourpy-1.1.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66544f853bfa85c0d07a68f6c648b2ec81dafd30f272565c37ab47a33b220684"}, + {file = "contourpy-1.1.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0c02b75acfea5cab07585d25069207e478d12309557f90a61b5a3b4f77f46ce"}, + {file = "contourpy-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41339b24471c58dc1499e56783fedc1afa4bb018bcd035cfb0ee2ad2a7501ef8"}, + {file = "contourpy-1.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f29fb0b3f1217dfe9362ec55440d0743fe868497359f2cf93293f4b2701b8251"}, + {file = "contourpy-1.1.1-cp38-cp38-win32.whl", hash = "sha256:f9dc7f933975367251c1b34da882c4f0e0b2e24bb35dc906d2f598a40b72bfc7"}, + {file = "contourpy-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:498e53573e8b94b1caeb9e62d7c2d053c263ebb6aa259c81050766beb50ff8d9"}, + {file = "contourpy-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ba42e3810999a0ddd0439e6e5dbf6d034055cdc72b7c5c839f37a7c274cb4eba"}, + {file = "contourpy-1.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c06e4c6e234fcc65435223c7b2a90f286b7f1b2733058bdf1345d218cc59e34"}, + {file = "contourpy-1.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca6fab080484e419528e98624fb5c4282148b847e3602dc8dbe0cb0669469887"}, + {file = "contourpy-1.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93df44ab351119d14cd1e6b52a5063d3336f0754b72736cc63db59307dabb718"}, + {file = "contourpy-1.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eafbef886566dc1047d7b3d4b14db0d5b7deb99638d8e1be4e23a7c7ac59ff0f"}, + {file = "contourpy-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efe0fab26d598e1ec07d72cf03eaeeba8e42b4ecf6b9ccb5a356fde60ff08b85"}, + {file = "contourpy-1.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f08e469821a5e4751c97fcd34bcb586bc243c39c2e39321822060ba902eac49e"}, + {file = "contourpy-1.1.1-cp39-cp39-win32.whl", hash = "sha256:bfc8a5e9238232a45ebc5cb3bfee71f1167064c8d382cadd6076f0d51cff1da0"}, + {file = "contourpy-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:c84fdf3da00c2827d634de4fcf17e3e067490c4aea82833625c4c8e6cdea0887"}, + {file = "contourpy-1.1.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:229a25f68046c5cf8067d6d6351c8b99e40da11b04d8416bf8d2b1d75922521e"}, + {file = "contourpy-1.1.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a10dab5ea1bd4401c9483450b5b0ba5416be799bbd50fc7a6cc5e2a15e03e8a3"}, + {file = "contourpy-1.1.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4f9147051cb8fdb29a51dc2482d792b3b23e50f8f57e3720ca2e3d438b7adf23"}, + {file = "contourpy-1.1.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a75cc163a5f4531a256f2c523bd80db509a49fc23721b36dd1ef2f60ff41c3cb"}, + {file = "contourpy-1.1.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b53d5769aa1f2d4ea407c65f2d1d08002952fac1d9e9d307aa2e1023554a163"}, + {file = "contourpy-1.1.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:11b836b7dbfb74e049c302bbf74b4b8f6cb9d0b6ca1bf86cfa8ba144aedadd9c"}, + {file = "contourpy-1.1.1.tar.gz", hash = "sha256:96ba37c2e24b7212a77da85004c38e7c4d155d3e72a45eeaf22c1f03f607e8ab"}, ] corner = [ {file = "corner-1.0.3.tar.gz", hash = "sha256:8330a1f44ecf98ba6a74b866351758e4ee5d7b9264988165f36757429b4dc4e9"}, @@ -1435,36 +1459,36 @@ coverage = [ {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, ] cycler = [ - {file = "cycler-0.11.0-py3-none-any.whl", hash = "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3"}, - {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, + {file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"}, + {file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"}, ] decorator = [ {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, ] distlib = [ - {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, - {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, ] emcee = [ {file = "emcee-3.1.4-py2.py3-none-any.whl", hash = "sha256:13d216a96b4f60661839d5f6ec482eda9e86b499502cda2bf6d8cc839ccf3e59"}, {file = "emcee-3.1.4.tar.gz", hash = "sha256:8e0e19dc8bcef9c6d02f860bef8ddc6c876b8878a6ce666943e2c5cfd9317fed"}, ] execnet = [ - {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, - {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, + {file = "execnet-2.0.2-py3-none-any.whl", hash = "sha256:88256416ae766bc9e8895c76a87928c0012183da3cc4fc18016e6f050e025f41"}, + {file = "execnet-2.0.2.tar.gz", hash = "sha256:cc59bc4423742fd71ad227122eb0dd44db51efb3dc4095b45ac9a08c770096af"}, ] executing = [ - {file = "executing-1.2.0-py2.py3-none-any.whl", hash = "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc"}, - {file = "executing-1.2.0.tar.gz", hash = "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107"}, + {file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"}, + {file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"}, ] filelock = [ - {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, - {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, ] flask = [ - {file = "Flask-2.3.2-py3-none-any.whl", hash = "sha256:77fd4e1249d8c9923de34907236b747ced06e5467ecac1a7bb7115ae0e9670b0"}, - {file = "Flask-2.3.2.tar.gz", hash = "sha256:8c2f9abd47a9e8df7f0c3f091ce9497d011dc3b31effcf4c85a6e2b50f4114ef"}, + {file = "flask-3.0.0-py3-none-any.whl", hash = "sha256:21128f47e4e3b9d597a3e8521a329bf56909b690fcc3fa3e477725aa81367638"}, + {file = "flask-3.0.0.tar.gz", hash = "sha256:cfadcdb638b609361d29ec22360d6070a77d7463dcb3ab08d2c2f2f168845f58"}, ] flask-restful = [ {file = "Flask-RESTful-0.3.10.tar.gz", hash = "sha256:fe4af2ef0027df8f9b4f797aba20c5566801b6ade995ac63b588abf1a59cec37"}, @@ -1474,60 +1498,68 @@ flatdict = [ {file = "flatdict-4.0.1.tar.gz", hash = "sha256:cd32f08fd31ed21eb09ebc76f06b6bd12046a24f77beb1fd0281917e47f26742"}, ] fonttools = [ - {file = "fonttools-4.40.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b802dcbf9bcff74672f292b2466f6589ab8736ce4dcf36f48eb994c2847c4b30"}, - {file = "fonttools-4.40.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f6e3fa3da923063c286320e728ba2270e49c73386e3a711aa680f4b0747d692"}, - {file = "fonttools-4.40.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fdf60f8a5c6bcce7d024a33f7e4bc7921f5b74e8ea13bccd204f2c8b86f3470"}, - {file = "fonttools-4.40.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91784e21a1a085fac07c6a407564f4a77feb471b5954c9ee55a4f9165151f6c1"}, - {file = "fonttools-4.40.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:05171f3c546f64d78569f10adc0de72561882352cac39ec7439af12304d8d8c0"}, - {file = "fonttools-4.40.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7449e5e306f3a930a8944c85d0cbc8429cba13503372a1a40f23124d6fb09b58"}, - {file = "fonttools-4.40.0-cp310-cp310-win32.whl", hash = "sha256:bae8c13abbc2511e9a855d2142c0ab01178dd66b1a665798f357da0d06253e0d"}, - {file = "fonttools-4.40.0-cp310-cp310-win_amd64.whl", hash = "sha256:425b74a608427499b0e45e433c34ddc350820b6f25b7c8761963a08145157a66"}, - {file = "fonttools-4.40.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:00ab569b2a3e591e00425023ade87e8fef90380c1dde61be7691cb524ca5f743"}, - {file = "fonttools-4.40.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:18ea64ac43e94c9e0c23d7a9475f1026be0e25b10dda8f236fc956188761df97"}, - {file = "fonttools-4.40.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:022c4a16b412293e7f1ce21b8bab7a6f9d12c4ffdf171fdc67122baddb973069"}, - {file = "fonttools-4.40.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:530c5d35109f3e0cea2535742d6a3bc99c0786cf0cbd7bb2dc9212387f0d908c"}, - {file = "fonttools-4.40.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5e00334c66f4e83535384cb5339526d01d02d77f142c23b2f97bd6a4f585497a"}, - {file = "fonttools-4.40.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb52c10fda31159c22c7ed85074e05f8b97da8773ea461706c273e31bcbea836"}, - {file = "fonttools-4.40.0-cp311-cp311-win32.whl", hash = "sha256:6a8d71b9a5c884c72741868e845c0e563c5d83dcaf10bb0ceeec3b4b2eb14c67"}, - {file = "fonttools-4.40.0-cp311-cp311-win_amd64.whl", hash = "sha256:15abb3d055c1b2dff9ce376b6c3db10777cb74b37b52b78f61657634fd348a0d"}, - {file = "fonttools-4.40.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14037c31138fbd21847ad5e5441dfdde003e0a8f3feb5812a1a21fd1c255ffbd"}, - {file = "fonttools-4.40.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:94c915f6716589f78bc00fbc14c5b8de65cfd11ee335d32504f1ef234524cb24"}, - {file = "fonttools-4.40.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37467cee0f32cada2ec08bc16c9c31f9b53ea54b2f5604bf25a1246b5f50593a"}, - {file = "fonttools-4.40.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56d4d85f5374b45b08d2f928517d1e313ea71b4847240398decd0ab3ebbca885"}, - {file = "fonttools-4.40.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8c4305b171b61040b1ee75d18f9baafe58bd3b798d1670078efe2c92436bfb63"}, - {file = "fonttools-4.40.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a954b90d1473c85a22ecf305761d9fd89da93bbd31dae86e7dea436ad2cb5dc9"}, - {file = "fonttools-4.40.0-cp38-cp38-win32.whl", hash = "sha256:1bc4c5b147be8dbc5df9cc8ac5e93ee914ad030fe2a201cc8f02f499db71011d"}, - {file = "fonttools-4.40.0-cp38-cp38-win_amd64.whl", hash = "sha256:8a917828dbfdb1cbe50cf40eeae6fbf9c41aef9e535649ed8f4982b2ef65c091"}, - {file = "fonttools-4.40.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:882983279bf39afe4e945109772c2ffad2be2c90983d6559af8b75c19845a80a"}, - {file = "fonttools-4.40.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c55f1b4109dbc3aeb496677b3e636d55ef46dc078c2a5e3f3db4e90f1c6d2907"}, - {file = "fonttools-4.40.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec468c022d09f1817c691cf884feb1030ef6f1e93e3ea6831b0d8144c06480d1"}, - {file = "fonttools-4.40.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d5adf4ba114f028fc3f5317a221fd8b0f4ef7a2e5524a2b1e0fd891b093791a"}, - {file = "fonttools-4.40.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aa83b3f151bc63970f39b2b42a06097c5a22fd7ed9f7ba008e618de4503d3895"}, - {file = "fonttools-4.40.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:97d95b8301b62bdece1af943b88bcb3680fd385f88346a4a899ee145913b414a"}, - {file = "fonttools-4.40.0-cp39-cp39-win32.whl", hash = "sha256:1a003608400dd1cca3e089e8c94973c6b51a4fb1ef00ff6d7641617b9242e637"}, - {file = "fonttools-4.40.0-cp39-cp39-win_amd64.whl", hash = "sha256:7961575221e3da0841c75da53833272c520000d76f7f71274dbf43370f8a1065"}, - {file = "fonttools-4.40.0-py3-none-any.whl", hash = "sha256:200729d12461e2038700d31f0d49ad5a7b55855dec7525074979a06b46f88505"}, - {file = "fonttools-4.40.0.tar.gz", hash = "sha256:337b6e83d7ee73c40ea62407f2ce03b07c3459e213b6f332b94a69923b9e1cb9"}, + {file = "fonttools-4.47.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3b629108351d25512d4ea1a8393a2dba325b7b7d7308116b605ea3f8e1be88df"}, + {file = "fonttools-4.47.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c19044256c44fe299d9a73456aabee4b4d06c6b930287be93b533b4737d70aa1"}, + {file = "fonttools-4.47.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8be28c036b9f186e8c7eaf8a11b42373e7e4949f9e9f370202b9da4c4c3f56c"}, + {file = "fonttools-4.47.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f83a4daef6d2a202acb9bf572958f91cfde5b10c8ee7fb1d09a4c81e5d851fd8"}, + {file = "fonttools-4.47.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4a5a5318ba5365d992666ac4fe35365f93004109d18858a3e18ae46f67907670"}, + {file = "fonttools-4.47.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8f57ecd742545362a0f7186774b2d1c53423ed9ece67689c93a1055b236f638c"}, + {file = "fonttools-4.47.2-cp310-cp310-win32.whl", hash = "sha256:a1c154bb85dc9a4cf145250c88d112d88eb414bad81d4cb524d06258dea1bdc0"}, + {file = "fonttools-4.47.2-cp310-cp310-win_amd64.whl", hash = "sha256:3e2b95dce2ead58fb12524d0ca7d63a63459dd489e7e5838c3cd53557f8933e1"}, + {file = "fonttools-4.47.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:29495d6d109cdbabe73cfb6f419ce67080c3ef9ea1e08d5750240fd4b0c4763b"}, + {file = "fonttools-4.47.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0a1d313a415eaaba2b35d6cd33536560deeebd2ed758b9bfb89ab5d97dc5deac"}, + {file = "fonttools-4.47.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90f898cdd67f52f18049250a6474185ef6544c91f27a7bee70d87d77a8daf89c"}, + {file = "fonttools-4.47.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3480eeb52770ff75140fe7d9a2ec33fb67b07efea0ab5129c7e0c6a639c40c70"}, + {file = "fonttools-4.47.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0255dbc128fee75fb9be364806b940ed450dd6838672a150d501ee86523ac61e"}, + {file = "fonttools-4.47.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f791446ff297fd5f1e2247c188de53c1bfb9dd7f0549eba55b73a3c2087a2703"}, + {file = "fonttools-4.47.2-cp311-cp311-win32.whl", hash = "sha256:740947906590a878a4bde7dd748e85fefa4d470a268b964748403b3ab2aeed6c"}, + {file = "fonttools-4.47.2-cp311-cp311-win_amd64.whl", hash = "sha256:63fbed184979f09a65aa9c88b395ca539c94287ba3a364517698462e13e457c9"}, + {file = "fonttools-4.47.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4ec558c543609e71b2275c4894e93493f65d2f41c15fe1d089080c1d0bb4d635"}, + {file = "fonttools-4.47.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e040f905d542362e07e72e03612a6270c33d38281fd573160e1003e43718d68d"}, + {file = "fonttools-4.47.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6dd58cc03016b281bd2c74c84cdaa6bd3ce54c5a7f47478b7657b930ac3ed8eb"}, + {file = "fonttools-4.47.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32ab2e9702dff0dd4510c7bb958f265a8d3dd5c0e2547e7b5f7a3df4979abb07"}, + {file = "fonttools-4.47.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a808f3c1d1df1f5bf39be869b6e0c263570cdafb5bdb2df66087733f566ea71"}, + {file = "fonttools-4.47.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac71e2e201df041a2891067dc36256755b1229ae167edbdc419b16da78732c2f"}, + {file = "fonttools-4.47.2-cp312-cp312-win32.whl", hash = "sha256:69731e8bea0578b3c28fdb43dbf95b9386e2d49a399e9a4ad736b8e479b08085"}, + {file = "fonttools-4.47.2-cp312-cp312-win_amd64.whl", hash = "sha256:b3e1304e5f19ca861d86a72218ecce68f391646d85c851742d265787f55457a4"}, + {file = "fonttools-4.47.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:254d9a6f7be00212bf0c3159e0a420eb19c63793b2c05e049eb337f3023c5ecc"}, + {file = "fonttools-4.47.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:eabae77a07c41ae0b35184894202305c3ad211a93b2eb53837c2a1143c8bc952"}, + {file = "fonttools-4.47.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86a5ab2873ed2575d0fcdf1828143cfc6b977ac448e3dc616bb1e3d20efbafa"}, + {file = "fonttools-4.47.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13819db8445a0cec8c3ff5f243af6418ab19175072a9a92f6cc8ca7d1452754b"}, + {file = "fonttools-4.47.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4e743935139aa485fe3253fc33fe467eab6ea42583fa681223ea3f1a93dd01e6"}, + {file = "fonttools-4.47.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d49ce3ea7b7173faebc5664872243b40cf88814ca3eb135c4a3cdff66af71946"}, + {file = "fonttools-4.47.2-cp38-cp38-win32.whl", hash = "sha256:94208ea750e3f96e267f394d5588579bb64cc628e321dbb1d4243ffbc291b18b"}, + {file = "fonttools-4.47.2-cp38-cp38-win_amd64.whl", hash = "sha256:0f750037e02beb8b3569fbff701a572e62a685d2a0e840d75816592280e5feae"}, + {file = "fonttools-4.47.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3d71606c9321f6701642bd4746f99b6089e53d7e9817fc6b964e90d9c5f0ecc6"}, + {file = "fonttools-4.47.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:86e0427864c6c91cf77f16d1fb9bf1bbf7453e824589e8fb8461b6ee1144f506"}, + {file = "fonttools-4.47.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a00bd0e68e88987dcc047ea31c26d40a3c61185153b03457956a87e39d43c37"}, + {file = "fonttools-4.47.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5d77479fb885ef38a16a253a2f4096bc3d14e63a56d6246bfdb56365a12b20c"}, + {file = "fonttools-4.47.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5465df494f20a7d01712b072ae3ee9ad2887004701b95cb2cc6dcb9c2c97a899"}, + {file = "fonttools-4.47.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4c811d3c73b6abac275babb8aa439206288f56fdb2c6f8835e3d7b70de8937a7"}, + {file = "fonttools-4.47.2-cp39-cp39-win32.whl", hash = "sha256:5b60e3afa9635e3dfd3ace2757039593e3bd3cf128be0ddb7a1ff4ac45fa5a50"}, + {file = "fonttools-4.47.2-cp39-cp39-win_amd64.whl", hash = "sha256:7ee48bd9d6b7e8f66866c9090807e3a4a56cf43ffad48962725a190e0dd774c8"}, + {file = "fonttools-4.47.2-py3-none-any.whl", hash = "sha256:7eb7ad665258fba68fd22228a09f347469d95a97fb88198e133595947a20a184"}, + {file = "fonttools-4.47.2.tar.gz", hash = "sha256:7df26dd3650e98ca45f1e29883c96a0b9f5bb6af8d632a6a108bc744fa0bd9b3"}, ] hypothesis = [ {file = "hypothesis-5.49.0-py3-none-any.whl", hash = "sha256:e91111f2f01abf2566041c4c86366aa7f08bfd5b3d858cc77a545fcf67df335e"}, {file = "hypothesis-5.49.0.tar.gz", hash = "sha256:36a4d5587c34193125d654b61bf9284e24a227d1edd339c49143378658a10c7d"}, ] identify = [ - {file = "identify-2.5.24-py2.py3-none-any.whl", hash = "sha256:986dbfb38b1140e763e413e6feb44cd731faf72d1909543178aa79b0e258265d"}, - {file = "identify-2.5.24.tar.gz", hash = "sha256:0aac67d5b4812498056d28a9a512a483f5085cc28640b02b258a59dac34301d4"}, + {file = "identify-2.5.33-py2.py3-none-any.whl", hash = "sha256:d40ce5fcd762817627670da8a7d8d8e65f24342d14539c59488dc603bf662e34"}, + {file = "identify-2.5.33.tar.gz", hash = "sha256:161558f9fe4559e1557e1bff323e8631f6a0e4837f7497767c1782832f16b62d"}, ] idna = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] importlib-metadata = [ - {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"}, - {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"}, + {file = "importlib_metadata-7.0.1-py3-none-any.whl", hash = "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e"}, + {file = "importlib_metadata-7.0.1.tar.gz", hash = "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"}, ] importlib-resources = [ - {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, - {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, + {file = "importlib_resources-6.1.1-py3-none-any.whl", hash = "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6"}, + {file = "importlib_resources-6.1.1.tar.gz", hash = "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a"}, ] iniconfig = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, @@ -1546,165 +1578,202 @@ itsdangerous = [ {file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, ] jedi = [ - {file = "jedi-0.18.2-py2.py3-none-any.whl", hash = "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e"}, - {file = "jedi-0.18.2.tar.gz", hash = "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"}, + {file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"}, + {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"}, ] jinja2 = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, ] joblib = [ - {file = "joblib-1.3.1-py3-none-any.whl", hash = "sha256:89cf0529520e01b3de7ac7b74a8102c90d16d54c64b5dd98cafcd14307fdf915"}, - {file = "joblib-1.3.1.tar.gz", hash = "sha256:1f937906df65329ba98013dc9692fe22a4c5e4a648112de500508b18a21b41e3"}, + {file = "joblib-1.3.2-py3-none-any.whl", hash = "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"}, + {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, ] kiwisolver = [ - {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6"}, - {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:10ee06759482c78bdb864f4109886dff7b8a56529bc1609d4f1112b93fe6423c"}, - {file = "kiwisolver-1.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c79ebe8f3676a4c6630fd3f777f3cfecf9289666c84e775a67d1d358578dc2e3"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:abbe9fa13da955feb8202e215c4018f4bb57469b1b78c7a4c5c7b93001699938"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7577c1987baa3adc4b3c62c33bd1118c3ef5c8ddef36f0f2c950ae0b199e100d"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8ad8285b01b0d4695102546b342b493b3ccc6781fc28c8c6a1bb63e95d22f09"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ed58b8acf29798b036d347791141767ccf65eee7f26bde03a71c944449e53de"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a68b62a02953b9841730db7797422f983935aeefceb1679f0fc85cbfbd311c32"}, - {file = "kiwisolver-1.4.4-cp310-cp310-win32.whl", hash = "sha256:e92a513161077b53447160b9bd8f522edfbed4bd9759e4c18ab05d7ef7e49408"}, - {file = "kiwisolver-1.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:3fe20f63c9ecee44560d0e7f116b3a747a5d7203376abeea292ab3152334d004"}, - {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e0ea21f66820452a3f5d1655f8704a60d66ba1191359b96541eaf457710a5fc6"}, - {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bc9db8a3efb3e403e4ecc6cd9489ea2bac94244f80c78e27c31dcc00d2790ac2"}, - {file = "kiwisolver-1.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d5b61785a9ce44e5a4b880272baa7cf6c8f48a5180c3e81c59553ba0cb0821ca"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2dbb44c3f7e6c4d3487b31037b1bdbf424d97687c1747ce4ff2895795c9bf69"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6295ecd49304dcf3bfbfa45d9a081c96509e95f4b9d0eb7ee4ec0530c4a96514"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4bd472dbe5e136f96a4b18f295d159d7f26fd399136f5b17b08c4e5f498cd494"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf7d9fce9bcc4752ca4a1b80aabd38f6d19009ea5cbda0e0856983cf6d0023f5"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d6601aed50c74e0ef02f4204da1816147a6d3fbdc8b3872d263338a9052c51"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:877272cf6b4b7e94c9614f9b10140e198d2186363728ed0f701c6eee1baec1da"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:db608a6757adabb32f1cfe6066e39b3706d8c3aa69bbc353a5b61edad36a5cb4"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5853eb494c71e267912275e5586fe281444eb5e722de4e131cddf9d442615626"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f0a1dbdb5ecbef0d34eb77e56fcb3e95bbd7e50835d9782a45df81cc46949750"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:283dffbf061a4ec60391d51e6155e372a1f7a4f5b15d59c8505339454f8989e4"}, - {file = "kiwisolver-1.4.4-cp311-cp311-win32.whl", hash = "sha256:d06adcfa62a4431d404c31216f0f8ac97397d799cd53800e9d3efc2fbb3cf14e"}, - {file = "kiwisolver-1.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e7da3fec7408813a7cebc9e4ec55afed2d0fd65c4754bc376bf03498d4e92686"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:62ac9cc684da4cf1778d07a89bf5f81b35834cb96ca523d3a7fb32509380cbf6"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41dae968a94b1ef1897cb322b39360a0812661dba7c682aa45098eb8e193dbdf"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02f79693ec433cb4b5f51694e8477ae83b3205768a6fb48ffba60549080e295b"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0611a0a2a518464c05ddd5a3a1a0e856ccc10e67079bb17f265ad19ab3c7597"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:db5283d90da4174865d520e7366801a93777201e91e79bacbac6e6927cbceede"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1041feb4cda8708ce73bb4dcb9ce1ccf49d553bf87c3954bdfa46f0c3f77252c"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-win32.whl", hash = "sha256:a553dadda40fef6bfa1456dc4be49b113aa92c2a9a9e8711e955618cd69622e3"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:03baab2d6b4a54ddbb43bba1a3a2d1627e82d205c5cf8f4c924dc49284b87166"}, - {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:841293b17ad704d70c578f1f0013c890e219952169ce8a24ebc063eecf775454"}, - {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f4f270de01dd3e129a72efad823da90cc4d6aafb64c410c9033aba70db9f1ff0"}, - {file = "kiwisolver-1.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f9f39e2f049db33a908319cf46624a569b36983c7c78318e9726a4cb8923b26c"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c97528e64cb9ebeff9701e7938653a9951922f2a38bd847787d4a8e498cc83ae"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d1573129aa0fd901076e2bfb4275a35f5b7aa60fbfb984499d661ec950320b0"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad881edc7ccb9d65b0224f4e4d05a1e85cf62d73aab798943df6d48ab0cd79a1"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b428ef021242344340460fa4c9185d0b1f66fbdbfecc6c63eff4b7c29fad429d"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2e407cb4bd5a13984a6c2c0fe1845e4e41e96f183e5e5cd4d77a857d9693494c"}, - {file = "kiwisolver-1.4.4-cp38-cp38-win32.whl", hash = "sha256:75facbe9606748f43428fc91a43edb46c7ff68889b91fa31f53b58894503a191"}, - {file = "kiwisolver-1.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:5bce61af018b0cb2055e0e72e7d65290d822d3feee430b7b8203d8a855e78766"}, - {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8c808594c88a025d4e322d5bb549282c93c8e1ba71b790f539567932722d7bd8"}, - {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0a71d85ecdd570ded8ac3d1c0f480842f49a40beb423bb8014539a9f32a5897"}, - {file = "kiwisolver-1.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b533558eae785e33e8c148a8d9921692a9fe5aa516efbdff8606e7d87b9d5824"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:efda5fc8cc1c61e4f639b8067d118e742b812c930f708e6667a5ce0d13499e29"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7c43e1e1206cd421cd92e6b3280d4385d41d7166b3ed577ac20444b6995a445f"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc8d3bd6c72b2dd9decf16ce70e20abcb3274ba01b4e1c96031e0c4067d1e7cd"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ea39b0ccc4f5d803e3337dd46bcce60b702be4d86fd0b3d7531ef10fd99a1ac"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968f44fdbf6dd757d12920d63b566eeb4d5b395fd2d00d29d7ef00a00582aac9"}, - {file = "kiwisolver-1.4.4-cp39-cp39-win32.whl", hash = "sha256:da7e547706e69e45d95e116e6939488d62174e033b763ab1496b4c29b76fabea"}, - {file = "kiwisolver-1.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:ba59c92039ec0a66103b1d5fe588fa546373587a7d68f5c96f743c3396afc04b"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:91672bacaa030f92fc2f43b620d7b337fd9a5af28b0d6ed3f77afc43c4a64b5a"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:787518a6789009c159453da4d6b683f468ef7a65bbde796bcea803ccf191058d"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da152d8cdcab0e56e4f45eb08b9aea6455845ec83172092f09b0e077ece2cf7a"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ecb1fa0db7bf4cff9dac752abb19505a233c7f16684c5826d1f11ebd9472b871"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:28bc5b299f48150b5f822ce68624e445040595a4ac3d59251703779836eceff9"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:81e38381b782cc7e1e46c4e14cd997ee6040768101aefc8fa3c24a4cc58e98f8"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2a66fdfb34e05b705620dd567f5a03f239a088d5a3f321e7b6ac3239d22aa286"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872b8ca05c40d309ed13eb2e582cab0c5a05e81e987ab9c521bf05ad1d5cf5cb"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:70e7c2e7b750585569564e2e5ca9845acfaa5da56ac46df68414f29fea97be9f"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9f85003f5dfa867e86d53fac6f7e6f30c045673fa27b603c397753bebadc3008"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e307eb9bd99801f82789b44bb45e9f541961831c7311521b13a6c85afc09767"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1792d939ec70abe76f5054d3f36ed5656021dcad1322d1cc996d4e54165cef9"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6cb459eea32a4e2cf18ba5fcece2dbdf496384413bc1bae15583f19e567f3b2"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36dafec3d6d6088d34e2de6b85f9d8e2324eb734162fba59d2ba9ed7a2043d5b"}, - {file = "kiwisolver-1.4.4.tar.gz", hash = "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955"}, + {file = "kiwisolver-1.4.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:05703cf211d585109fcd72207a31bb170a0f22144d68298dc5e61b3c946518af"}, + {file = "kiwisolver-1.4.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:146d14bebb7f1dc4d5fbf74f8a6cb15ac42baadee8912eb84ac0b3b2a3dc6ac3"}, + {file = "kiwisolver-1.4.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ef7afcd2d281494c0a9101d5c571970708ad911d028137cd558f02b851c08b4"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9eaa8b117dc8337728e834b9c6e2611f10c79e38f65157c4c38e9400286f5cb1"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ec20916e7b4cbfb1f12380e46486ec4bcbaa91a9c448b97023fde0d5bbf9e4ff"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39b42c68602539407884cf70d6a480a469b93b81b7701378ba5e2328660c847a"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa12042de0171fad672b6c59df69106d20d5596e4f87b5e8f76df757a7c399aa"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a40773c71d7ccdd3798f6489aaac9eee213d566850a9533f8d26332d626b82c"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:19df6e621f6d8b4b9c4d45f40a66839294ff2bb235e64d2178f7522d9170ac5b"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:83d78376d0d4fd884e2c114d0621624b73d2aba4e2788182d286309ebdeed770"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e391b1f0a8a5a10ab3b9bb6afcfd74f2175f24f8975fb87ecae700d1503cdee0"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:852542f9481f4a62dbb5dd99e8ab7aedfeb8fb6342349a181d4036877410f525"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59edc41b24031bc25108e210c0def6f6c2191210492a972d585a06ff246bb79b"}, + {file = "kiwisolver-1.4.5-cp310-cp310-win32.whl", hash = "sha256:a6aa6315319a052b4ee378aa171959c898a6183f15c1e541821c5c59beaa0238"}, + {file = "kiwisolver-1.4.5-cp310-cp310-win_amd64.whl", hash = "sha256:d0ef46024e6a3d79c01ff13801cb19d0cad7fd859b15037aec74315540acc276"}, + {file = "kiwisolver-1.4.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:11863aa14a51fd6ec28688d76f1735f8f69ab1fabf388851a595d0721af042f5"}, + {file = "kiwisolver-1.4.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8ab3919a9997ab7ef2fbbed0cc99bb28d3c13e6d4b1ad36e97e482558a91be90"}, + {file = "kiwisolver-1.4.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fcc700eadbbccbf6bc1bcb9dbe0786b4b1cb91ca0dcda336eef5c2beed37b797"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dfdd7c0b105af050eb3d64997809dc21da247cf44e63dc73ff0fd20b96be55a9"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76c6a5964640638cdeaa0c359382e5703e9293030fe730018ca06bc2010c4437"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bbea0db94288e29afcc4c28afbf3a7ccaf2d7e027489c449cf7e8f83c6346eb9"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ceec1a6bc6cab1d6ff5d06592a91a692f90ec7505d6463a88a52cc0eb58545da"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:040c1aebeda72197ef477a906782b5ab0d387642e93bda547336b8957c61022e"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f91de7223d4c7b793867797bacd1ee53bfe7359bd70d27b7b58a04efbb9436c8"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:faae4860798c31530dd184046a900e652c95513796ef51a12bc086710c2eec4d"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0157420efcb803e71d1b28e2c287518b8808b7cf1ab8af36718fd0a2c453eb0"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:06f54715b7737c2fecdbf140d1afb11a33d59508a47bf11bb38ecf21dc9ab79f"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fdb7adb641a0d13bdcd4ef48e062363d8a9ad4a182ac7647ec88f695e719ae9f"}, + {file = "kiwisolver-1.4.5-cp311-cp311-win32.whl", hash = "sha256:bb86433b1cfe686da83ce32a9d3a8dd308e85c76b60896d58f082136f10bffac"}, + {file = "kiwisolver-1.4.5-cp311-cp311-win_amd64.whl", hash = "sha256:6c08e1312a9cf1074d17b17728d3dfce2a5125b2d791527f33ffbe805200a355"}, + {file = "kiwisolver-1.4.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:32d5cf40c4f7c7b3ca500f8985eb3fb3a7dfc023215e876f207956b5ea26632a"}, + {file = "kiwisolver-1.4.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f846c260f483d1fd217fe5ed7c173fb109efa6b1fc8381c8b7552c5781756192"}, + {file = "kiwisolver-1.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5ff5cf3571589b6d13bfbfd6bcd7a3f659e42f96b5fd1c4830c4cf21d4f5ef45"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7269d9e5f1084a653d575c7ec012ff57f0c042258bf5db0954bf551c158466e7"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da802a19d6e15dffe4b0c24b38b3af68e6c1a68e6e1d8f30148c83864f3881db"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3aba7311af82e335dd1e36ffff68aaca609ca6290c2cb6d821a39aa075d8e3ff"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:763773d53f07244148ccac5b084da5adb90bfaee39c197554f01b286cf869228"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2270953c0d8cdab5d422bee7d2007f043473f9d2999631c86a223c9db56cbd16"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d099e745a512f7e3bbe7249ca835f4d357c586d78d79ae8f1dcd4d8adeb9bda9"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:74db36e14a7d1ce0986fa104f7d5637aea5c82ca6326ed0ec5694280942d1162"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e5bab140c309cb3a6ce373a9e71eb7e4873c70c2dda01df6820474f9889d6d4"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0f114aa76dc1b8f636d077979c0ac22e7cd8f3493abbab152f20eb8d3cda71f3"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:88a2df29d4724b9237fc0c6eaf2a1adae0cdc0b3e9f4d8e7dc54b16812d2d81a"}, + {file = "kiwisolver-1.4.5-cp312-cp312-win32.whl", hash = "sha256:72d40b33e834371fd330fb1472ca19d9b8327acb79a5821d4008391db8e29f20"}, + {file = "kiwisolver-1.4.5-cp312-cp312-win_amd64.whl", hash = "sha256:2c5674c4e74d939b9d91dda0fae10597ac7521768fec9e399c70a1f27e2ea2d9"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3a2b053a0ab7a3960c98725cfb0bf5b48ba82f64ec95fe06f1d06c99b552e130"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cd32d6c13807e5c66a7cbb79f90b553642f296ae4518a60d8d76243b0ad2898"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59ec7b7c7e1a61061850d53aaf8e93db63dce0c936db1fda2658b70e4a1be709"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da4cfb373035def307905d05041c1d06d8936452fe89d464743ae7fb8371078b"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2400873bccc260b6ae184b2b8a4fec0e4082d30648eadb7c3d9a13405d861e89"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1b04139c4236a0f3aff534479b58f6f849a8b351e1314826c2d230849ed48985"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:4e66e81a5779b65ac21764c295087de82235597a2293d18d943f8e9e32746265"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7931d8f1f67c4be9ba1dd9c451fb0eeca1a25b89e4d3f89e828fe12a519b782a"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b3f7e75f3015df442238cca659f8baa5f42ce2a8582727981cbfa15fee0ee205"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:bbf1d63eef84b2e8c89011b7f2235b1e0bf7dacc11cac9431fc6468e99ac77fb"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4c380469bd3f970ef677bf2bcba2b6b0b4d5c75e7a020fb863ef75084efad66f"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-win32.whl", hash = "sha256:9408acf3270c4b6baad483865191e3e582b638b1654a007c62e3efe96f09a9a3"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-win_amd64.whl", hash = "sha256:5b94529f9b2591b7af5f3e0e730a4e0a41ea174af35a4fd067775f9bdfeee01a"}, + {file = "kiwisolver-1.4.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:11c7de8f692fc99816e8ac50d1d1aef4f75126eefc33ac79aac02c099fd3db71"}, + {file = "kiwisolver-1.4.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:53abb58632235cd154176ced1ae8f0d29a6657aa1aa9decf50b899b755bc2b93"}, + {file = "kiwisolver-1.4.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:88b9f257ca61b838b6f8094a62418421f87ac2a1069f7e896c36a7d86b5d4c29"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3195782b26fc03aa9c6913d5bad5aeb864bdc372924c093b0f1cebad603dd712"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc579bf0f502e54926519451b920e875f433aceb4624a3646b3252b5caa9e0b6"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a580c91d686376f0f7c295357595c5a026e6cbc3d77b7c36e290201e7c11ecb"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cfe6ab8da05c01ba6fbea630377b5da2cd9bcbc6338510116b01c1bc939a2c18"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:d2e5a98f0ec99beb3c10e13b387f8db39106d53993f498b295f0c914328b1333"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a51a263952b1429e429ff236d2f5a21c5125437861baeed77f5e1cc2d2c7c6da"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3edd2fa14e68c9be82c5b16689e8d63d89fe927e56debd6e1dbce7a26a17f81b"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:74d1b44c6cfc897df648cc9fdaa09bc3e7679926e6f96df05775d4fb3946571c"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:76d9289ed3f7501012e05abb8358bbb129149dbd173f1f57a1bf1c22d19ab7cc"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:92dea1ffe3714fa8eb6a314d2b3c773208d865a0e0d35e713ec54eea08a66250"}, + {file = "kiwisolver-1.4.5-cp38-cp38-win32.whl", hash = "sha256:5c90ae8c8d32e472be041e76f9d2f2dbff4d0b0be8bd4041770eddb18cf49a4e"}, + {file = "kiwisolver-1.4.5-cp38-cp38-win_amd64.whl", hash = "sha256:c7940c1dc63eb37a67721b10d703247552416f719c4188c54e04334321351ced"}, + {file = "kiwisolver-1.4.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9407b6a5f0d675e8a827ad8742e1d6b49d9c1a1da5d952a67d50ef5f4170b18d"}, + {file = "kiwisolver-1.4.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15568384086b6df3c65353820a4473575dbad192e35010f622c6ce3eebd57af9"}, + {file = "kiwisolver-1.4.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0dc9db8e79f0036e8173c466d21ef18e1befc02de8bf8aa8dc0813a6dc8a7046"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cdc8a402aaee9a798b50d8b827d7ecf75edc5fb35ea0f91f213ff927c15f4ff0"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6c3bd3cde54cafb87d74d8db50b909705c62b17c2099b8f2e25b461882e544ff"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:955e8513d07a283056b1396e9a57ceddbd272d9252c14f154d450d227606eb54"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:346f5343b9e3f00b8db8ba359350eb124b98c99efd0b408728ac6ebf38173958"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9098e0049e88c6a24ff64545cdfc50807818ba6c1b739cae221bbbcbc58aad3"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:00bd361b903dc4bbf4eb165f24d1acbee754fce22ded24c3d56eec268658a5cf"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7b8b454bac16428b22560d0a1cf0a09875339cab69df61d7805bf48919415901"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f1d072c2eb0ad60d4c183f3fb44ac6f73fb7a8f16a2694a91f988275cbf352f9"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:31a82d498054cac9f6d0b53d02bb85811185bcb477d4b60144f915f3b3126342"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6512cb89e334e4700febbffaaa52761b65b4f5a3cf33f960213d5656cea36a77"}, + {file = "kiwisolver-1.4.5-cp39-cp39-win32.whl", hash = "sha256:9db8ea4c388fdb0f780fe91346fd438657ea602d58348753d9fb265ce1bca67f"}, + {file = "kiwisolver-1.4.5-cp39-cp39-win_amd64.whl", hash = "sha256:59415f46a37f7f2efeec758353dd2eae1b07640d8ca0f0c42548ec4125492635"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5c7b3b3a728dc6faf3fc372ef24f21d1e3cee2ac3e9596691d746e5a536de920"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:620ced262a86244e2be10a676b646f29c34537d0d9cc8eb26c08f53d98013390"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:378a214a1e3bbf5ac4a8708304318b4f890da88c9e6a07699c4ae7174c09a68d"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf7be1207676ac608a50cd08f102f6742dbfc70e8d60c4db1c6897f62f71523"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ba55dce0a9b8ff59495ddd050a0225d58bd0983d09f87cfe2b6aec4f2c1234e4"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fd32ea360bcbb92d28933fc05ed09bffcb1704ba3fc7942e81db0fd4f81a7892"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5e7139af55d1688f8b960ee9ad5adafc4ac17c1c473fe07133ac092310d76544"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dced8146011d2bc2e883f9bd68618b8247387f4bbec46d7392b3c3b032640126"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9bf3325c47b11b2e51bca0824ea217c7cd84491d8ac4eefd1e409705ef092bd"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5794cf59533bc3f1b1c821f7206a3617999db9fbefc345360aafe2e067514929"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e368f200bbc2e4f905b8e71eb38b3c04333bddaa6a2464a6355487b02bb7fb09"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5d706eba36b4c4d5bc6c6377bb6568098765e990cfc21ee16d13963fab7b3e7"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85267bd1aa8880a9c88a8cb71e18d3d64d2751a790e6ca6c27b8ccc724bcd5ad"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210ef2c3a1f03272649aff1ef992df2e724748918c4bc2d5a90352849eb40bea"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:11d011a7574eb3b82bcc9c1a1d35c1d7075677fdd15de527d91b46bd35e935ee"}, + {file = "kiwisolver-1.4.5.tar.gz", hash = "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec"}, ] lxml = [ - {file = "lxml-4.9.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:76cf573e5a365e790396a5cc2b909812633409306c6531a6877c59061e42c4f2"}, - {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b1f42b6921d0e81b1bcb5e395bc091a70f41c4d4e55ba99c6da2b31626c44892"}, - {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9f102706d0ca011de571de32c3247c6476b55bb6bc65a20f682f000b07a4852a"}, - {file = "lxml-4.9.2-cp27-cp27m-win32.whl", hash = "sha256:8d0b4612b66ff5d62d03bcaa043bb018f74dfea51184e53f067e6fdcba4bd8de"}, - {file = "lxml-4.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:4c8f293f14abc8fd3e8e01c5bd86e6ed0b6ef71936ded5bf10fe7a5efefbaca3"}, - {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2899456259589aa38bfb018c364d6ae7b53c5c22d8e27d0ec7609c2a1ff78b50"}, - {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6749649eecd6a9871cae297bffa4ee76f90b4504a2a2ab528d9ebe912b101975"}, - {file = "lxml-4.9.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a08cff61517ee26cb56f1e949cca38caabe9ea9fbb4b1e10a805dc39844b7d5c"}, - {file = "lxml-4.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:85cabf64adec449132e55616e7ca3e1000ab449d1d0f9d7f83146ed5bdcb6d8a"}, - {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8340225bd5e7a701c0fa98284c849c9b9fc9238abf53a0ebd90900f25d39a4e4"}, - {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:1ab8f1f932e8f82355e75dda5413a57612c6ea448069d4fb2e217e9a4bed13d4"}, - {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:699a9af7dffaf67deeae27b2112aa06b41c370d5e7633e0ee0aea2e0b6c211f7"}, - {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9cc34af337a97d470040f99ba4282f6e6bac88407d021688a5d585e44a23184"}, - {file = "lxml-4.9.2-cp310-cp310-win32.whl", hash = "sha256:d02a5399126a53492415d4906ab0ad0375a5456cc05c3fc0fc4ca11771745cda"}, - {file = "lxml-4.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:a38486985ca49cfa574a507e7a2215c0c780fd1778bb6290c21193b7211702ab"}, - {file = "lxml-4.9.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c83203addf554215463b59f6399835201999b5e48019dc17f182ed5ad87205c9"}, - {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2a87fa548561d2f4643c99cd13131acb607ddabb70682dcf1dff5f71f781a4bf"}, - {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:d6b430a9938a5a5d85fc107d852262ddcd48602c120e3dbb02137c83d212b380"}, - {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3efea981d956a6f7173b4659849f55081867cf897e719f57383698af6f618a92"}, - {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:df0623dcf9668ad0445e0558a21211d4e9a149ea8f5666917c8eeec515f0a6d1"}, - {file = "lxml-4.9.2-cp311-cp311-win32.whl", hash = "sha256:da248f93f0418a9e9d94b0080d7ebc407a9a5e6d0b57bb30db9b5cc28de1ad33"}, - {file = "lxml-4.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:3818b8e2c4b5148567e1b09ce739006acfaa44ce3156f8cbbc11062994b8e8dd"}, - {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca989b91cf3a3ba28930a9fc1e9aeafc2a395448641df1f387a2d394638943b0"}, - {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:822068f85e12a6e292803e112ab876bc03ed1f03dddb80154c395f891ca6b31e"}, - {file = "lxml-4.9.2-cp35-cp35m-win32.whl", hash = "sha256:be7292c55101e22f2a3d4d8913944cbea71eea90792bf914add27454a13905df"}, - {file = "lxml-4.9.2-cp35-cp35m-win_amd64.whl", hash = "sha256:998c7c41910666d2976928c38ea96a70d1aa43be6fe502f21a651e17483a43c5"}, - {file = "lxml-4.9.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:b26a29f0b7fc6f0897f043ca366142d2b609dc60756ee6e4e90b5f762c6adc53"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:ab323679b8b3030000f2be63e22cdeea5b47ee0abd2d6a1dc0c8103ddaa56cd7"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:689bb688a1db722485e4610a503e3e9210dcc20c520b45ac8f7533c837be76fe"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f49e52d174375a7def9915c9f06ec4e569d235ad428f70751765f48d5926678c"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:36c3c175d34652a35475a73762b545f4527aec044910a651d2bf50de9c3352b1"}, - {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a35f8b7fa99f90dd2f5dc5a9fa12332642f087a7641289ca6c40d6e1a2637d8e"}, - {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:58bfa3aa19ca4c0f28c5dde0ff56c520fbac6f0daf4fac66ed4c8d2fb7f22e74"}, - {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc718cd47b765e790eecb74d044cc8d37d58562f6c314ee9484df26276d36a38"}, - {file = "lxml-4.9.2-cp36-cp36m-win32.whl", hash = "sha256:d5bf6545cd27aaa8a13033ce56354ed9e25ab0e4ac3b5392b763d8d04b08e0c5"}, - {file = "lxml-4.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:3ab9fa9d6dc2a7f29d7affdf3edebf6ece6fb28a6d80b14c3b2fb9d39b9322c3"}, - {file = "lxml-4.9.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:05ca3f6abf5cf78fe053da9b1166e062ade3fa5d4f92b4ed688127ea7d7b1d03"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:a5da296eb617d18e497bcf0a5c528f5d3b18dadb3619fbdadf4ed2356ef8d941"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:04876580c050a8c5341d706dd464ff04fd597095cc8c023252566a8826505726"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c9ec3eaf616d67db0764b3bb983962b4f385a1f08304fd30c7283954e6a7869b"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2a29ba94d065945944016b6b74e538bdb1751a1db6ffb80c9d3c2e40d6fa9894"}, - {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a82d05da00a58b8e4c0008edbc8a4b6ec5a4bc1e2ee0fb6ed157cf634ed7fa45"}, - {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:223f4232855ade399bd409331e6ca70fb5578efef22cf4069a6090acc0f53c0e"}, - {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d17bc7c2ccf49c478c5bdd447594e82692c74222698cfc9b5daae7ae7e90743b"}, - {file = "lxml-4.9.2-cp37-cp37m-win32.whl", hash = "sha256:b64d891da92e232c36976c80ed7ebb383e3f148489796d8d31a5b6a677825efe"}, - {file = "lxml-4.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a0a336d6d3e8b234a3aae3c674873d8f0e720b76bc1d9416866c41cd9500ffb9"}, - {file = "lxml-4.9.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:da4dd7c9c50c059aba52b3524f84d7de956f7fef88f0bafcf4ad7dde94a064e8"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:821b7f59b99551c69c85a6039c65b75f5683bdc63270fec660f75da67469ca24"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:e5168986b90a8d1f2f9dc1b841467c74221bd752537b99761a93d2d981e04889"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8e20cb5a47247e383cf4ff523205060991021233ebd6f924bca927fcf25cf86f"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:13598ecfbd2e86ea7ae45ec28a2a54fb87ee9b9fdb0f6d343297d8e548392c03"}, - {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:880bbbcbe2fca64e2f4d8e04db47bcdf504936fa2b33933efd945e1b429bea8c"}, - {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7d2278d59425777cfcb19735018d897ca8303abe67cc735f9f97177ceff8027f"}, - {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5344a43228767f53a9df6e5b253f8cdca7dfc7b7aeae52551958192f56d98457"}, - {file = "lxml-4.9.2-cp38-cp38-win32.whl", hash = "sha256:925073b2fe14ab9b87e73f9a5fde6ce6392da430f3004d8b72cc86f746f5163b"}, - {file = "lxml-4.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:9b22c5c66f67ae00c0199f6055705bc3eb3fcb08d03d2ec4059a2b1b25ed48d7"}, - {file = "lxml-4.9.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5f50a1c177e2fa3ee0667a5ab79fdc6b23086bc8b589d90b93b4bd17eb0e64d1"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:090c6543d3696cbe15b4ac6e175e576bcc3f1ccfbba970061b7300b0c15a2140"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:63da2ccc0857c311d764e7d3d90f429c252e83b52d1f8f1d1fe55be26827d1f4"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:5b4545b8a40478183ac06c073e81a5ce4cf01bf1734962577cf2bb569a5b3bbf"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2e430cd2824f05f2d4f687701144556646bae8f249fd60aa1e4c768ba7018947"}, - {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6804daeb7ef69e7b36f76caddb85cccd63d0c56dedb47555d2fc969e2af6a1a5"}, - {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a6e441a86553c310258aca15d1c05903aaf4965b23f3bc2d55f200804e005ee5"}, - {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ca34efc80a29351897e18888c71c6aca4a359247c87e0b1c7ada14f0ab0c0fb2"}, - {file = "lxml-4.9.2-cp39-cp39-win32.whl", hash = "sha256:6b418afe5df18233fc6b6093deb82a32895b6bb0b1155c2cdb05203f583053f1"}, - {file = "lxml-4.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1496ea22ca2c830cbcbd473de8f114a320da308438ae65abad6bab7867fe38f"}, - {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b264171e3143d842ded311b7dccd46ff9ef34247129ff5bf5066123c55c2431c"}, - {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0dc313ef231edf866912e9d8f5a042ddab56c752619e92dfd3a2c277e6a7299a"}, - {file = "lxml-4.9.2-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:16efd54337136e8cd72fb9485c368d91d77a47ee2d42b057564aae201257d419"}, - {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0f2b1e0d79180f344ff9f321327b005ca043a50ece8713de61d1cb383fb8ac05"}, - {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:7b770ed79542ed52c519119473898198761d78beb24b107acf3ad65deae61f1f"}, - {file = "lxml-4.9.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efa29c2fe6b4fdd32e8ef81c1528506895eca86e1d8c4657fda04c9b3786ddf9"}, - {file = "lxml-4.9.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7e91ee82f4199af8c43d8158024cbdff3d931df350252288f0d4ce656df7f3b5"}, - {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b23e19989c355ca854276178a0463951a653309fb8e57ce674497f2d9f208746"}, - {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:01d36c05f4afb8f7c20fd9ed5badca32a2029b93b1750f571ccc0b142531caf7"}, - {file = "lxml-4.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7b515674acfdcadb0eb5d00d8a709868173acece5cb0be3dd165950cbfdf5409"}, - {file = "lxml-4.9.2.tar.gz", hash = "sha256:2455cfaeb7ac70338b3257f41e21f0724f4b5b0c0e7702da67ee6c3640835b67"}, + {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e"}, + {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da"}, + {file = "lxml-5.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c"}, + {file = "lxml-5.1.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb"}, + {file = "lxml-5.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2befa20a13f1a75c751f47e00929fb3433d67eb9923c2c0b364de449121f447c"}, + {file = "lxml-5.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22b7ee4c35f374e2c20337a95502057964d7e35b996b1c667b5c65c567d2252a"}, + {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf8443781533b8d37b295016a4b53c1494fa9a03573c09ca5104550c138d5c05"}, + {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147"}, + {file = "lxml-5.1.0-cp310-cp310-win32.whl", hash = "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93"}, + {file = "lxml-5.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a"}, + {file = "lxml-5.1.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa"}, + {file = "lxml-5.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af8920ce4a55ff41167ddbc20077f5698c2e710ad3353d32a07d3264f3a2021e"}, + {file = "lxml-5.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7cfced4a069003d8913408e10ca8ed092c49a7f6cefee9bb74b6b3e860683b45"}, + {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9e5ac3437746189a9b4121db2a7b86056ac8786b12e88838696899328fc44bb2"}, + {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204"}, + {file = "lxml-5.1.0-cp311-cp311-win32.whl", hash = "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b"}, + {file = "lxml-5.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e"}, + {file = "lxml-5.1.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a"}, + {file = "lxml-5.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16dd953fb719f0ffc5bc067428fc9e88f599e15723a85618c45847c96f11f431"}, + {file = "lxml-5.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16018f7099245157564d7148165132c70adb272fb5a17c048ba70d9cc542a1a1"}, + {file = "lxml-5.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:82cd34f1081ae4ea2ede3d52f71b7be313756e99b4b5f829f89b12da552d3aa3"}, + {file = "lxml-5.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:19a1bc898ae9f06bccb7c3e1dfd73897ecbbd2c96afe9095a6026016e5ca97b8"}, + {file = "lxml-5.1.0-cp312-cp312-win32.whl", hash = "sha256:13521a321a25c641b9ea127ef478b580b5ec82aa2e9fc076c86169d161798b01"}, + {file = "lxml-5.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:1ad17c20e3666c035db502c78b86e58ff6b5991906e55bdbef94977700c72623"}, + {file = "lxml-5.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:24ef5a4631c0b6cceaf2dbca21687e29725b7c4e171f33a8f8ce23c12558ded1"}, + {file = "lxml-5.1.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d2900b7f5318bc7ad8631d3d40190b95ef2aa8cc59473b73b294e4a55e9f30f"}, + {file = "lxml-5.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:601f4a75797d7a770daed8b42b97cd1bb1ba18bd51a9382077a6a247a12aa38d"}, + {file = "lxml-5.1.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4b68c961b5cc402cbd99cca5eb2547e46ce77260eb705f4d117fd9c3f932b95"}, + {file = "lxml-5.1.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:afd825e30f8d1f521713a5669b63657bcfe5980a916c95855060048b88e1adb7"}, + {file = "lxml-5.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:262bc5f512a66b527d026518507e78c2f9c2bd9eb5c8aeeb9f0eb43fcb69dc67"}, + {file = "lxml-5.1.0-cp36-cp36m-win32.whl", hash = "sha256:e856c1c7255c739434489ec9c8aa9cdf5179785d10ff20add308b5d673bed5cd"}, + {file = "lxml-5.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c7257171bb8d4432fe9d6fdde4d55fdbe663a63636a17f7f9aaba9bcb3153ad7"}, + {file = "lxml-5.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b9e240ae0ba96477682aa87899d94ddec1cc7926f9df29b1dd57b39e797d5ab5"}, + {file = "lxml-5.1.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a96f02ba1bcd330807fc060ed91d1f7a20853da6dd449e5da4b09bfcc08fdcf5"}, + {file = "lxml-5.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3898ae2b58eeafedfe99e542a17859017d72d7f6a63de0f04f99c2cb125936"}, + {file = "lxml-5.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61c5a7edbd7c695e54fca029ceb351fc45cd8860119a0f83e48be44e1c464862"}, + {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3aeca824b38ca78d9ee2ab82bd9883083d0492d9d17df065ba3b94e88e4d7ee6"}, + {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764"}, + {file = "lxml-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8"}, + {file = "lxml-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5"}, + {file = "lxml-5.1.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84"}, + {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa"}, + {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45"}, + {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:98f3f020a2b736566c707c8e034945c02aa94e124c24f77ca097c446f81b01f1"}, + {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e"}, + {file = "lxml-5.1.0-cp38-cp38-win32.whl", hash = "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a"}, + {file = "lxml-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3"}, + {file = "lxml-5.1.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581"}, + {file = "lxml-5.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f8b0c78e7aac24979ef09b7f50da871c2de2def043d468c4b41f512d831e912"}, + {file = "lxml-5.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9bcf86dfc8ff3e992fed847c077bd875d9e0ba2fa25d859c3a0f0f76f07f0c8d"}, + {file = "lxml-5.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:49a9b4af45e8b925e1cd6f3b15bbba2c81e7dba6dce170c677c9cda547411e14"}, + {file = "lxml-5.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:280f3edf15c2a967d923bcfb1f8f15337ad36f93525828b40a0f9d6c2ad24890"}, + {file = "lxml-5.1.0-cp39-cp39-win32.whl", hash = "sha256:ed7326563024b6e91fef6b6c7a1a2ff0a71b97793ac33dbbcf38f6005e51ff6e"}, + {file = "lxml-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:8d7b4beebb178e9183138f552238f7e6613162a42164233e2bda00cb3afac58f"}, + {file = "lxml-5.1.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9bd0ae7cc2b85320abd5e0abad5ccee5564ed5f0cc90245d2f9a8ef330a8deae"}, + {file = "lxml-5.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8c1d679df4361408b628f42b26a5d62bd3e9ba7f0c0e7969f925021554755aa"}, + {file = "lxml-5.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2ad3a8ce9e8a767131061a22cd28fdffa3cd2dc193f399ff7b81777f3520e372"}, + {file = "lxml-5.1.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:304128394c9c22b6569eba2a6d98392b56fbdfbad58f83ea702530be80d0f9df"}, + {file = "lxml-5.1.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d74fcaf87132ffc0447b3c685a9f862ffb5b43e70ea6beec2fb8057d5d2a1fea"}, + {file = "lxml-5.1.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:8cf5877f7ed384dabfdcc37922c3191bf27e55b498fecece9fd5c2c7aaa34c33"}, + {file = "lxml-5.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:877efb968c3d7eb2dad540b6cabf2f1d3c0fbf4b2d309a3c141f79c7e0061324"}, + {file = "lxml-5.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f14a4fb1c1c402a22e6a341a24c1341b4a3def81b41cd354386dcb795f83897"}, + {file = "lxml-5.1.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:25663d6e99659544ee8fe1b89b1a8c0aaa5e34b103fab124b17fa958c4a324a6"}, + {file = "lxml-5.1.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8b9f19df998761babaa7f09e6bc169294eefafd6149aaa272081cbddc7ba4ca3"}, + {file = "lxml-5.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e53d7e6a98b64fe54775d23a7c669763451340c3d44ad5e3a3b48a1efbdc96f"}, + {file = "lxml-5.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c3cd1fc1dc7c376c54440aeaaa0dcc803d2126732ff5c6b68ccd619f2e64be4f"}, + {file = "lxml-5.1.0.tar.gz", hash = "sha256:3eea6ed6e6c918e468e693c41ef07f3c3acc310b70ddd9cc72d9ef84bc9564ca"}, ] markupsafe = [ {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, @@ -1727,6 +1796,16 @@ markupsafe = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1759,47 +1838,53 @@ markupsafe = [ {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, ] matplotlib = [ - {file = "matplotlib-3.7.1-cp310-cp310-macosx_10_12_universal2.whl", hash = "sha256:95cbc13c1fc6844ab8812a525bbc237fa1470863ff3dace7352e910519e194b1"}, - {file = "matplotlib-3.7.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:08308bae9e91aca1ec6fd6dda66237eef9f6294ddb17f0d0b3c863169bf82353"}, - {file = "matplotlib-3.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:544764ba51900da4639c0f983b323d288f94f65f4024dc40ecb1542d74dc0500"}, - {file = "matplotlib-3.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56d94989191de3fcc4e002f93f7f1be5da476385dde410ddafbb70686acf00ea"}, - {file = "matplotlib-3.7.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99bc9e65901bb9a7ce5e7bb24af03675cbd7c70b30ac670aa263240635999a4"}, - {file = "matplotlib-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb7d248c34a341cd4c31a06fd34d64306624c8cd8d0def7abb08792a5abfd556"}, - {file = "matplotlib-3.7.1-cp310-cp310-win32.whl", hash = "sha256:ce463ce590f3825b52e9fe5c19a3c6a69fd7675a39d589e8b5fbe772272b3a24"}, - {file = "matplotlib-3.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:3d7bc90727351fb841e4d8ae620d2d86d8ed92b50473cd2b42ce9186104ecbba"}, - {file = "matplotlib-3.7.1-cp311-cp311-macosx_10_12_universal2.whl", hash = "sha256:770a205966d641627fd5cf9d3cb4b6280a716522cd36b8b284a8eb1581310f61"}, - {file = "matplotlib-3.7.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f67bfdb83a8232cb7a92b869f9355d677bce24485c460b19d01970b64b2ed476"}, - {file = "matplotlib-3.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2bf092f9210e105f414a043b92af583c98f50050559616930d884387d0772aba"}, - {file = "matplotlib-3.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89768d84187f31717349c6bfadc0e0d8c321e8eb34522acec8a67b1236a66332"}, - {file = "matplotlib-3.7.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83111e6388dec67822e2534e13b243cc644c7494a4bb60584edbff91585a83c6"}, - {file = "matplotlib-3.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a867bf73a7eb808ef2afbca03bcdb785dae09595fbe550e1bab0cd023eba3de0"}, - {file = "matplotlib-3.7.1-cp311-cp311-win32.whl", hash = "sha256:fbdeeb58c0cf0595efe89c05c224e0a502d1aa6a8696e68a73c3efc6bc354304"}, - {file = "matplotlib-3.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:c0bd19c72ae53e6ab979f0ac6a3fafceb02d2ecafa023c5cca47acd934d10be7"}, - {file = "matplotlib-3.7.1-cp38-cp38-macosx_10_12_universal2.whl", hash = "sha256:6eb88d87cb2c49af00d3bbc33a003f89fd9f78d318848da029383bfc08ecfbfb"}, - {file = "matplotlib-3.7.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:cf0e4f727534b7b1457898c4f4ae838af1ef87c359b76dcd5330fa31893a3ac7"}, - {file = "matplotlib-3.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:46a561d23b91f30bccfd25429c3c706afe7d73a5cc64ef2dfaf2b2ac47c1a5dc"}, - {file = "matplotlib-3.7.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8704726d33e9aa8a6d5215044b8d00804561971163563e6e6591f9dcf64340cc"}, - {file = "matplotlib-3.7.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4cf327e98ecf08fcbb82685acaf1939d3338548620ab8dfa02828706402c34de"}, - {file = "matplotlib-3.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:617f14ae9d53292ece33f45cba8503494ee199a75b44de7717964f70637a36aa"}, - {file = "matplotlib-3.7.1-cp38-cp38-win32.whl", hash = "sha256:7c9a4b2da6fac77bcc41b1ea95fadb314e92508bf5493ceff058e727e7ecf5b0"}, - {file = "matplotlib-3.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:14645aad967684e92fc349493fa10c08a6da514b3d03a5931a1bac26e6792bd1"}, - {file = "matplotlib-3.7.1-cp39-cp39-macosx_10_12_universal2.whl", hash = "sha256:81a6b377ea444336538638d31fdb39af6be1a043ca5e343fe18d0f17e098770b"}, - {file = "matplotlib-3.7.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:28506a03bd7f3fe59cd3cd4ceb2a8d8a2b1db41afede01f66c42561b9be7b4b7"}, - {file = "matplotlib-3.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8c587963b85ce41e0a8af53b9b2de8dddbf5ece4c34553f7bd9d066148dc719c"}, - {file = "matplotlib-3.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8bf26ade3ff0f27668989d98c8435ce9327d24cffb7f07d24ef609e33d582439"}, - {file = "matplotlib-3.7.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:def58098f96a05f90af7e92fd127d21a287068202aa43b2a93476170ebd99e87"}, - {file = "matplotlib-3.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f883a22a56a84dba3b588696a2b8a1ab0d2c3d41be53264115c71b0a942d8fdb"}, - {file = "matplotlib-3.7.1-cp39-cp39-win32.whl", hash = "sha256:4f99e1b234c30c1e9714610eb0c6d2f11809c9c78c984a613ae539ea2ad2eb4b"}, - {file = "matplotlib-3.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:3ba2af245e36990facf67fde840a760128ddd71210b2ab6406e640188d69d136"}, - {file = "matplotlib-3.7.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3032884084f541163f295db8a6536e0abb0db464008fadca6c98aaf84ccf4717"}, - {file = "matplotlib-3.7.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a2cb34336110e0ed8bb4f650e817eed61fa064acbefeb3591f1b33e3a84fd96"}, - {file = "matplotlib-3.7.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b867e2f952ed592237a1828f027d332d8ee219ad722345b79a001f49df0936eb"}, - {file = "matplotlib-3.7.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:57bfb8c8ea253be947ccb2bc2d1bb3862c2bccc662ad1b4626e1f5e004557042"}, - {file = "matplotlib-3.7.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:438196cdf5dc8d39b50a45cb6e3f6274edbcf2254f85fa9b895bf85851c3a613"}, - {file = "matplotlib-3.7.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:21e9cff1a58d42e74d01153360de92b326708fb205250150018a52c70f43c290"}, - {file = "matplotlib-3.7.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d4725d70b7c03e082bbb8a34639ede17f333d7247f56caceb3801cb6ff703d"}, - {file = "matplotlib-3.7.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:97cc368a7268141afb5690760921765ed34867ffb9655dd325ed207af85c7529"}, - {file = "matplotlib-3.7.1.tar.gz", hash = "sha256:7b73305f25eab4541bd7ee0b96d87e53ae9c9f1823be5659b806cd85786fe882"}, + {file = "matplotlib-3.7.4-cp310-cp310-macosx_10_12_universal2.whl", hash = "sha256:b71079239bd866bf56df023e5146de159cb0c7294e508830901f4d79e2d89385"}, + {file = "matplotlib-3.7.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:bf91a42f6274a64cb41189120b620c02e574535ff6671fa836cade7701b06fbd"}, + {file = "matplotlib-3.7.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f757e8b42841d6add0cb69b42497667f0d25a404dcd50bd923ec9904e38414c4"}, + {file = "matplotlib-3.7.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4dfee00aa4bd291e08bb9461831c26ce0da85ca9781bb8794f2025c6e925281"}, + {file = "matplotlib-3.7.4-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3640f33632beb3993b698b1be9d1c262b742761d6101f3c27b87b2185d25c875"}, + {file = "matplotlib-3.7.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff539c4a17ecdf076ed808ee271ffae4a30dcb7e157b99ccae2c837262c07db6"}, + {file = "matplotlib-3.7.4-cp310-cp310-win32.whl", hash = "sha256:24b8f28af3e766195c09b780b15aa9f6710192b415ae7866b9c03dee7ec86370"}, + {file = "matplotlib-3.7.4-cp310-cp310-win_amd64.whl", hash = "sha256:3fa193286712c3b6c3cfa5fe8a6bb563f8c52cc750006c782296e0807ce5e799"}, + {file = "matplotlib-3.7.4-cp311-cp311-macosx_10_12_universal2.whl", hash = "sha256:b167f54cb4654b210c9624ec7b54e2b3b8de68c93a14668937e7e53df60770ec"}, + {file = "matplotlib-3.7.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7dfe6821f1944cb35603ff22e21510941bbcce7ccf96095beffaac890d39ce77"}, + {file = "matplotlib-3.7.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3c557d9165320dff3c5f2bb99bfa0b6813d3e626423ff71c40d6bc23b83c3339"}, + {file = "matplotlib-3.7.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08372696b3bb45c563472a552a705bfa0942f0a8ffe084db8a4e8f9153fbdf9d"}, + {file = "matplotlib-3.7.4-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81e1a7ac818000e8ac3ca696c3fdc501bc2d3adc89005e7b4e22ee5e9d51de98"}, + {file = "matplotlib-3.7.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:390920a3949906bc4b0216198d378f2a640c36c622e3584dd0c79a7c59ae9f50"}, + {file = "matplotlib-3.7.4-cp311-cp311-win32.whl", hash = "sha256:62e094d8da26294634da9e7f1856beee3978752b1b530c8e1763d2faed60cc10"}, + {file = "matplotlib-3.7.4-cp311-cp311-win_amd64.whl", hash = "sha256:f8fc2df756105784e650605e024d36dc2d048d68e5c1b26df97ee25d1bd41f9f"}, + {file = "matplotlib-3.7.4-cp312-cp312-macosx_10_12_universal2.whl", hash = "sha256:568574756127791903604e315c11aef9f255151e4cfe20ec603a70f9dda8e259"}, + {file = "matplotlib-3.7.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7d479aac338195e2199a8cfc03c4f2f55914e6a120177edae79e0340a6406457"}, + {file = "matplotlib-3.7.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:32183d4be84189a4c52b4b8861434d427d9118db2cec32986f98ed6c02dcfbb6"}, + {file = "matplotlib-3.7.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0037d066cca1f4bda626c507cddeb6f7da8283bc6a214da2db13ff2162933c52"}, + {file = "matplotlib-3.7.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44856632ebce88abd8efdc0a0dceec600418dcac06b72ae77af0019d260aa243"}, + {file = "matplotlib-3.7.4-cp312-cp312-win_amd64.whl", hash = "sha256:632fc938c22117d4241411191cfb88ac264a4c0a9ac702244641ddf30f0d739c"}, + {file = "matplotlib-3.7.4-cp38-cp38-macosx_10_12_universal2.whl", hash = "sha256:ce163be048613b9d1962273708cc97e09ca05d37312e670d166cf332b80bbaff"}, + {file = "matplotlib-3.7.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:e680f49bb8052ba3b2698e370155d2b4afb49f9af1cc611a26579d5981e2852a"}, + {file = "matplotlib-3.7.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0604880e4327114054199108b7390f987f4f40ee5ce728985836889e11a780ba"}, + {file = "matplotlib-3.7.4-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1e6abcde6fc52475f9d6a12b9f1792aee171ce7818ef6df5d61cb0b82816e6e8"}, + {file = "matplotlib-3.7.4-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f59a70e2ec3212033ef6633ed07682da03f5249379722512a3a2a26a7d9a738e"}, + {file = "matplotlib-3.7.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a9981b2a2dd9da06eca4ab5855d09b54b8ce7377c3e0e3957767b83219d652d"}, + {file = "matplotlib-3.7.4-cp38-cp38-win32.whl", hash = "sha256:83859ac26839660ecd164ee8311272074250b915ac300f9b2eccc84410f8953b"}, + {file = "matplotlib-3.7.4-cp38-cp38-win_amd64.whl", hash = "sha256:7a7709796ac59fe8debde68272388be6ed449c8971362eb5b60d280eac8dadde"}, + {file = "matplotlib-3.7.4-cp39-cp39-macosx_10_12_universal2.whl", hash = "sha256:b1d70bc1ea1bf110bec64f4578de3e14947909a8887df4c1fd44492eca487955"}, + {file = "matplotlib-3.7.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c83f49e795a5de6c168876eea723f5b88355202f9603c55977f5356213aa8280"}, + {file = "matplotlib-3.7.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5c9133f230945fe10652eb33e43642e933896194ef6a4f8d5e79bb722bdb2000"}, + {file = "matplotlib-3.7.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:798ff59022eeb276380ce9a73ba35d13c3d1499ab9b73d194fd07f1b0a41c304"}, + {file = "matplotlib-3.7.4-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1707b20b25e90538c2ce8d4409e30f0ef1df4017cc65ad0439633492a973635b"}, + {file = "matplotlib-3.7.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e6227ca8492baeef873cdd8e169a318efb5c3a25ce94e69727e7f964995b0b1"}, + {file = "matplotlib-3.7.4-cp39-cp39-win32.whl", hash = "sha256:5661c8639aded7d1bbf781373a359011cb1dd09199dee49043e9e68dd16f07ba"}, + {file = "matplotlib-3.7.4-cp39-cp39-win_amd64.whl", hash = "sha256:55eec941a4743f0bd3e5b8ee180e36b7ea8e62f867bf2613937c9f01b9ac06a2"}, + {file = "matplotlib-3.7.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ab16868714e5cc90ec8f7ff5d83d23bcd6559224d8e9cb5227c9f58748889fe8"}, + {file = "matplotlib-3.7.4-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c698b33f9a3f0b127a8e614c8fb4087563bb3caa9c9d95298722fa2400cdd3f"}, + {file = "matplotlib-3.7.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be3493bbcb4d255cb71de1f9050ac71682fce21a56089eadbcc8e21784cb12ee"}, + {file = "matplotlib-3.7.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:f8c725d1dd2901b2e7ec6cd64165e00da2978cc23d4143cb9ef745bec88e6b04"}, + {file = "matplotlib-3.7.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:286332f8f45f8ffde2d2119b9fdd42153dccd5025fa9f451b4a3b5c086e26da5"}, + {file = "matplotlib-3.7.4-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:116ef0b43aa00ff69260b4cce39c571e4b8c6f893795b708303fa27d9b9d7548"}, + {file = "matplotlib-3.7.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c90590d4b46458677d80bc3218f3f1ac11fc122baa9134e0cb5b3e8fc3714052"}, + {file = "matplotlib-3.7.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:de7c07069687be64fd9d119da3122ba13a8d399eccd3f844815f0dc78a870b2c"}, + {file = "matplotlib-3.7.4.tar.gz", hash = "sha256:7cd4fef8187d1dd0d9dcfdbaa06ac326d396fb8c71c647129f0bf56835d77026"}, ] matplotlib-inline = [ {file = "matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, @@ -1835,33 +1920,33 @@ mypy-extensions = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] netcdf4 = [ - {file = "netCDF4-1.6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59eba11eac34f9c7bd209121b26954c71ff69a2eb2838d2ac339b39ea60a95c0"}, - {file = "netCDF4-1.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:541286ce7318033b326da2e4325f08037b561887d5fd25d12c2b0cad74eb1edc"}, - {file = "netCDF4-1.6.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6620f7962daff02df2fbcbc8f7af53edf21512bc7f5697d846d424a094f96345"}, - {file = "netCDF4-1.6.4-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dcf74f7b15ae6105c09df93af22ad61366a288fbb329fdbe3b9606c56c788f5"}, - {file = "netCDF4-1.6.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b368fc185cac7d8890a8cf1016488cd252a144008144d06a615925b09bf8d67e"}, - {file = "netCDF4-1.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:eb19c51fa090b562be4f87ff736b629ffafc03d7bf1fa10f623f957d49417b4f"}, - {file = "netCDF4-1.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:08a7efab50d5a4da6227244e544a74f676b732ccdff2b8a4678174c15a3f8dff"}, - {file = "netCDF4-1.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9b7ad3f66d50970fa1bd5509c52bb8a99740c8a79fafa83bfd0bc7348b6ab5a"}, - {file = "netCDF4-1.6.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c5cad50d4e6e5a3b35c76862c58393ddf93baa44de98b6e040ac21896145881"}, - {file = "netCDF4-1.6.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c2395c53a37717b1047650e1c9de68a3d69542bb25df5c594e1e14e9480bb18"}, - {file = "netCDF4-1.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:c85f77599c61a88d512d6536e181ff1c01fd16f4740367e4f8e5cacb36500293"}, - {file = "netCDF4-1.6.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e3efefdb9fc936503e89ff5b40b33f2c96527bae1e9672a2d369c5f7cb30bbd"}, - {file = "netCDF4-1.6.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74a3e720a9764d13d669763c1f3c73f3f719ef181dccecad062440eea03f069e"}, - {file = "netCDF4-1.6.4-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c67bc7fafced3f9370fdef6ce7abe235a771bed7ebe534f4ba3c84d9689dae23"}, - {file = "netCDF4-1.6.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b599a3b5c0d0e3affa70d7954c2b0c4ab7d7bdb52b0e413c811da9725982de33"}, - {file = "netCDF4-1.6.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b21d47c23edd02ff83160c8ccc1e4d4946a91d454b246e7f63d7a6d63901707c"}, - {file = "netCDF4-1.6.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8eff69acdf250ebaf817cc9bbfb457d241f92f989c7f9cca33eb29985c0f9797"}, - {file = "netCDF4-1.6.4-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4d6bcdf27fd611eaec891f4281caf5c56bbad6f5fc245ce3e6dd2dc4fc3fad56"}, - {file = "netCDF4-1.6.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99a3d476242d489da26f71e318ddd314723ca766f4db11270863a764b107bc3"}, - {file = "netCDF4-1.6.4-cp38-cp38-win_amd64.whl", hash = "sha256:422077c7303cb17cbe8b9ace965cafda1f9d4c9b035809fc5c87091e7ff4d606"}, - {file = "netCDF4-1.6.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0050e49889c357ae0a73f96e2b9988f250396689b78142de88a39ac8b1e702aa"}, - {file = "netCDF4-1.6.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a2cd2e2a9da98f1838e1edfeb011b840a434e04d156052b936ffe49f5298126"}, - {file = "netCDF4-1.6.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:695cd0a40df49b4218350686c44e13b3a4671e53ee33faf7439a89940cbccc68"}, - {file = "netCDF4-1.6.4-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e582e6e41b27fc3a3296239afe065941bda60118d585df0ad41603f6f962888e"}, - {file = "netCDF4-1.6.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7b4ab3f0cd28a6685031e8b9be00c3047b4256da576c9de540440d013a31cea"}, - {file = "netCDF4-1.6.4-cp39-cp39-win_amd64.whl", hash = "sha256:1a950b3d8fcffca05e371a4cb64dc76580375588d1fca43b788192f409108ebe"}, - {file = "netCDF4-1.6.4.tar.gz", hash = "sha256:66da6542cbc7a6045cd1d979397dfd5a3f6c880c76d52b8f98bb108c82ee8c6e"}, + {file = "netCDF4-1.6.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d23b97cbde2bf413fadc4697c5c255a0436511c02f811e127e0fb12f5b882a4c"}, + {file = "netCDF4-1.6.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e5edfed673005f47f8d2fbea9c72c382b085dd358ac3c20ca743a563ed7b90e"}, + {file = "netCDF4-1.6.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10d2ac9ae1308ca837d86c6dc304ec455a85bdba0f2175e222844a54589168dc"}, + {file = "netCDF4-1.6.5-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a63a2be2f80977ac23bb0aa736c565011fd4639097ce0922e01b0dc38015df2"}, + {file = "netCDF4-1.6.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aaceea2097d292bad398d9f9b4fe403efa7b1568fcfa6faba9b67b1630027f9"}, + {file = "netCDF4-1.6.5-cp310-cp310-win_amd64.whl", hash = "sha256:111357d9e12eb79e8d58bfd91bc6b230d35b17a0ebd8c546d17416e8ceebea49"}, + {file = "netCDF4-1.6.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c5fede0b34c0a02a1b9e84116bfb3fcd2f80124a651d4836e72b785d10e2f15"}, + {file = "netCDF4-1.6.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3de5512b9270aa6472e4f3aa2bf895a7364c1d4f8667ce3b82e8232197d4fec8"}, + {file = "netCDF4-1.6.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b20971a164431f6eca1d24df8aa153db15c2c1b9630e83ccc5cf004e8ac8151d"}, + {file = "netCDF4-1.6.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad1101d538077152b866782e44458356981526bf2ea9cc07930bf28b589c82a7"}, + {file = "netCDF4-1.6.5-cp311-cp311-win_amd64.whl", hash = "sha256:de4dc973fae9e2bbdf42e094125e423a4c25393172a61958314969b055a38889"}, + {file = "netCDF4-1.6.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:19e16c63cdd7c0dbffe284a4a65f226ba1026f476f35cbedd099b4792b395f69"}, + {file = "netCDF4-1.6.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b994afce2ca4073f6b757385a6c0ffec25ecaae2b8821535b303c7cdbf6de42b"}, + {file = "netCDF4-1.6.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0187646e3348e7a8cd654617dda65517df138042c94c2fcc6682ff7c8c6654dc"}, + {file = "netCDF4-1.6.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1ab5dabac27d25fcc82c52dc29a74a6585e865208cce35f4e285df83d3df0b2"}, + {file = "netCDF4-1.6.5-cp312-cp312-win_amd64.whl", hash = "sha256:081e9043ac6160989f60570928eabe803c88ce7df1d3f79f2345dc48f68ef752"}, + {file = "netCDF4-1.6.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b47b22dda5b25ba6291f97634d7ac67b0a843f8ae5c9d9d5813c15364f66d0a"}, + {file = "netCDF4-1.6.5-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4609dd62d14798c9524327287091875449d68588c128abb768fc0c76c4a28165"}, + {file = "netCDF4-1.6.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2455e9d35fde067e6a6bdc24aa9d44962235a071cec49904d1589e298c23dcd3"}, + {file = "netCDF4-1.6.5-cp38-cp38-win_amd64.whl", hash = "sha256:2c210794d96431d92b5992e46ad8a9f97237bf6d6956f8816978a03dc0fa18c3"}, + {file = "netCDF4-1.6.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:18255b8b283d32d3900092f29c67e53aa25bd8f0dfe7adde59fe782d865a381c"}, + {file = "netCDF4-1.6.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:53050562bac84738bbd121fbbee9593d074579f5d6fdaafcb981abeb5c964225"}, + {file = "netCDF4-1.6.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:938c062382406bca9198b16adddd87c09b00521766b138cdfd11c95546eefeb8"}, + {file = "netCDF4-1.6.5-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a8300451d7542d3c4ff1dcccf5fb1c7d44bdd1dc08ec77dab04416caf13cb1f"}, + {file = "netCDF4-1.6.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a27db2701feef31201c9b20b04a9579196edc20dfc339ca423c7b81e462d6e14"}, + {file = "netCDF4-1.6.5-cp39-cp39-win_amd64.whl", hash = "sha256:574d7742ab321e5f9f33b5b1296c4ad4e5c469152c17d4fc453d5070e413e596"}, + {file = "netCDF4-1.6.5.tar.gz", hash = "sha256:824881d0aacfde5bd982d6adedd8574259c85553781e7b83e0ce82b890bfa0ef"}, ] networkx = [ {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, @@ -1902,8 +1987,8 @@ numpy = [ {file = "numpy-1.21.0.zip", hash = "sha256:e80fe25cba41c124d04c662f33f6364909b985f2eb5998aaa5ae4b9587242cce"}, ] packaging = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] pandas = [ {file = "pandas-1.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c81b8d91e9ae861eb4406b4e0f8d4dabbc105b9c479b3d1e921fba1d35b5b62a"}, @@ -1931,96 +2016,98 @@ parso = [ {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, ] pexpect = [ - {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, - {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, + {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, + {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, ] pickleshare = [ {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, ] pillow = [ - {file = "Pillow-9.5.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16"}, - {file = "Pillow-9.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ba1b81ee69573fe7124881762bb4cd2e4b6ed9dd28c9c60a632902fe8db8b38"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe7e1c262d3392afcf5071df9afa574544f28eac825284596ac6db56e6d11062"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f36397bf3f7d7c6a3abdea815ecf6fd14e7fcd4418ab24bae01008d8d8ca15e"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:252a03f1bdddce077eff2354c3861bf437c892fb1832f75ce813ee94347aa9b5"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:85ec677246533e27770b0de5cf0f9d6e4ec0c212a1f89dfc941b64b21226009d"}, - {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b416f03d37d27290cb93597335a2f85ed446731200705b22bb927405320de903"}, - {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1781a624c229cb35a2ac31cc4a77e28cafc8900733a864870c49bfeedacd106a"}, - {file = "Pillow-9.5.0-cp310-cp310-win32.whl", hash = "sha256:8507eda3cd0608a1f94f58c64817e83ec12fa93a9436938b191b80d9e4c0fc44"}, - {file = "Pillow-9.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:d3c6b54e304c60c4181da1c9dadf83e4a54fd266a99c70ba646a9baa626819eb"}, - {file = "Pillow-9.5.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:7ec6f6ce99dab90b52da21cf0dc519e21095e332ff3b399a357c187b1a5eee32"}, - {file = "Pillow-9.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:560737e70cb9c6255d6dcba3de6578a9e2ec4b573659943a5e7e4af13f298f5c"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96e88745a55b88a7c64fa49bceff363a1a27d9a64e04019c2281049444a571e3"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d9c206c29b46cfd343ea7cdfe1232443072bbb270d6a46f59c259460db76779a"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfcc2c53c06f2ccb8976fb5c71d448bdd0a07d26d8e07e321c103416444c7ad1"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:a0f9bb6c80e6efcde93ffc51256d5cfb2155ff8f78292f074f60f9e70b942d99"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8d935f924bbab8f0a9a28404422da8af4904e36d5c33fc6f677e4c4485515625"}, - {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fed1e1cf6a42577953abbe8e6cf2fe2f566daebde7c34724ec8803c4c0cda579"}, - {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296"}, - {file = "Pillow-9.5.0-cp311-cp311-win32.whl", hash = "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec"}, - {file = "Pillow-9.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4"}, - {file = "Pillow-9.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089"}, - {file = "Pillow-9.5.0-cp312-cp312-win32.whl", hash = "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb"}, - {file = "Pillow-9.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b"}, - {file = "Pillow-9.5.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:375f6e5ee9620a271acb6820b3d1e94ffa8e741c0601db4c0c4d3cb0a9c224bf"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99eb6cafb6ba90e436684e08dad8be1637efb71c4f2180ee6b8f940739406e78"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfaaf10b6172697b9bceb9a3bd7b951819d1ca339a5ef294d1f1ac6d7f63270"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:763782b2e03e45e2c77d7779875f4432e25121ef002a41829d8868700d119392"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:35f6e77122a0c0762268216315bf239cf52b88865bba522999dc38f1c52b9b47"}, - {file = "Pillow-9.5.0-cp37-cp37m-win32.whl", hash = "sha256:aca1c196f407ec7cf04dcbb15d19a43c507a81f7ffc45b690899d6a76ac9fda7"}, - {file = "Pillow-9.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322724c0032af6692456cd6ed554bb85f8149214d97398bb80613b04e33769f6"}, - {file = "Pillow-9.5.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:a0aa9417994d91301056f3d0038af1199eb7adc86e646a36b9e050b06f526597"}, - {file = "Pillow-9.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f8286396b351785801a976b1e85ea88e937712ee2c3ac653710a4a57a8da5d9c"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c830a02caeb789633863b466b9de10c015bded434deb3ec87c768e53752ad22a"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbd359831c1657d69bb81f0db962905ee05e5e9451913b18b831febfe0519082"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8fc330c3370a81bbf3f88557097d1ea26cd8b019d6433aa59f71195f5ddebbf"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:7002d0797a3e4193c7cdee3198d7c14f92c0836d6b4a3f3046a64bd1ce8df2bf"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:229e2c79c00e85989a34b5981a2b67aa079fd08c903f0aaead522a1d68d79e51"}, - {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9adf58f5d64e474bed00d69bcd86ec4bcaa4123bfa70a65ce72e424bfb88ed96"}, - {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:662da1f3f89a302cc22faa9f14a262c2e3951f9dbc9617609a47521c69dd9f8f"}, - {file = "Pillow-9.5.0-cp38-cp38-win32.whl", hash = "sha256:6608ff3bf781eee0cd14d0901a2b9cc3d3834516532e3bd673a0a204dc8615fc"}, - {file = "Pillow-9.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:e49eb4e95ff6fd7c0c402508894b1ef0e01b99a44320ba7d8ecbabefddcc5569"}, - {file = "Pillow-9.5.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:482877592e927fd263028c105b36272398e3e1be3269efda09f6ba21fd83ec66"}, - {file = "Pillow-9.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3ded42b9ad70e5f1754fb7c2e2d6465a9c842e41d178f262e08b8c85ed8a1d8e"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c446d2245ba29820d405315083d55299a796695d747efceb5717a8b450324115"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aca1152d93dcc27dc55395604dcfc55bed5f25ef4c98716a928bacba90d33a3"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:608488bdcbdb4ba7837461442b90ea6f3079397ddc968c31265c1e056964f1ef"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:60037a8db8750e474af7ffc9faa9b5859e6c6d0a50e55c45576bf28be7419705"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:07999f5834bdc404c442146942a2ecadd1cb6292f5229f4ed3b31e0a108746b1"}, - {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a127ae76092974abfbfa38ca2d12cbeddcdeac0fb71f9627cc1135bedaf9d51a"}, - {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:489f8389261e5ed43ac8ff7b453162af39c3e8abd730af8363587ba64bb2e865"}, - {file = "Pillow-9.5.0-cp39-cp39-win32.whl", hash = "sha256:9b1af95c3a967bf1da94f253e56b6286b50af23392a886720f563c547e48e964"}, - {file = "Pillow-9.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:77165c4a5e7d5a284f10a6efaa39a0ae8ba839da344f20b111d62cc932fa4e5d"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:833b86a98e0ede388fa29363159c9b1a294b0905b5128baf01db683672f230f5"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aaf305d6d40bd9632198c766fb64f0c1a83ca5b667f16c1e79e1661ab5060140"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0852ddb76d85f127c135b6dd1f0bb88dbb9ee990d2cd9aa9e28526c93e794fba"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:91ec6fe47b5eb5a9968c79ad9ed78c342b1f97a091677ba0e012701add857829"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cb841572862f629b99725ebaec3287fc6d275be9b14443ea746c1dd325053cbd"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c380b27d041209b849ed246b111b7c166ba36d7933ec6e41175fd15ab9eb1572"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c9af5a3b406a50e313467e3565fc99929717f780164fe6fbb7704edba0cebbe"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5671583eab84af046a397d6d0ba25343c00cd50bce03787948e0fff01d4fd9b1"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:84a6f19ce086c1bf894644b43cd129702f781ba5751ca8572f08aa40ef0ab7b7"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1e7723bd90ef94eda669a3c2c19d549874dd5badaeefabefd26053304abe5799"}, - {file = "Pillow-9.5.0.tar.gz", hash = "sha256:bf548479d336726d7a0eceb6e767e179fbde37833ae42794602631a070d630f1"}, + {file = "pillow-10.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:7823bdd049099efa16e4246bdf15e5a13dbb18a51b68fa06d6c1d4d8b99a796e"}, + {file = "pillow-10.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:83b2021f2ade7d1ed556bc50a399127d7fb245e725aa0113ebd05cfe88aaf588"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fad5ff2f13d69b7e74ce5b4ecd12cc0ec530fcee76356cac6742785ff71c452"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da2b52b37dad6d9ec64e653637a096905b258d2fc2b984c41ae7d08b938a67e4"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:47c0995fc4e7f79b5cfcab1fc437ff2890b770440f7696a3ba065ee0fd496563"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:322bdf3c9b556e9ffb18f93462e5f749d3444ce081290352c6070d014c93feb2"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:51f1a1bffc50e2e9492e87d8e09a17c5eea8409cda8d3f277eb6edc82813c17c"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:69ffdd6120a4737710a9eee73e1d2e37db89b620f702754b8f6e62594471dee0"}, + {file = "pillow-10.2.0-cp310-cp310-win32.whl", hash = "sha256:c6dafac9e0f2b3c78df97e79af707cdc5ef8e88208d686a4847bab8266870023"}, + {file = "pillow-10.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:aebb6044806f2e16ecc07b2a2637ee1ef67a11840a66752751714a0d924adf72"}, + {file = "pillow-10.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:7049e301399273a0136ff39b84c3678e314f2158f50f517bc50285fb5ec847ad"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35bb52c37f256f662abdfa49d2dfa6ce5d93281d323a9af377a120e89a9eafb5"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c23f307202661071d94b5e384e1e1dc7dfb972a28a2310e4ee16103e66ddb67"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:773efe0603db30c281521a7c0214cad7836c03b8ccff897beae9b47c0b657d61"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11fa2e5984b949b0dd6d7a94d967743d87c577ff0b83392f17cb3990d0d2fd6e"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:716d30ed977be8b37d3ef185fecb9e5a1d62d110dfbdcd1e2a122ab46fddb03f"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a086c2af425c5f62a65e12fbf385f7c9fcb8f107d0849dba5839461a129cf311"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c8de2789052ed501dd829e9cae8d3dcce7acb4777ea4a479c14521c942d395b1"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:609448742444d9290fd687940ac0b57fb35e6fd92bdb65386e08e99af60bf757"}, + {file = "pillow-10.2.0-cp311-cp311-win32.whl", hash = "sha256:823ef7a27cf86df6597fa0671066c1b596f69eba53efa3d1e1cb8b30f3533068"}, + {file = "pillow-10.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:1da3b2703afd040cf65ec97efea81cfba59cdbed9c11d8efc5ab09df9509fc56"}, + {file = "pillow-10.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:edca80cbfb2b68d7b56930b84a0e45ae1694aeba0541f798e908a49d66b837f1"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:1b5e1b74d1bd1b78bc3477528919414874748dd363e6272efd5abf7654e68bef"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0eae2073305f451d8ecacb5474997c08569fb4eb4ac231ffa4ad7d342fdc25ac"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7c2286c23cd350b80d2fc9d424fc797575fb16f854b831d16fd47ceec078f2c"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e23412b5c41e58cec602f1135c57dfcf15482013ce6e5f093a86db69646a5aa"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:52a50aa3fb3acb9cf7213573ef55d31d6eca37f5709c69e6858fe3bc04a5c2a2"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:127cee571038f252a552760076407f9cff79761c3d436a12af6000cd182a9d04"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8d12251f02d69d8310b046e82572ed486685c38f02176bd08baf216746eb947f"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:54f1852cd531aa981bc0965b7d609f5f6cc8ce8c41b1139f6ed6b3c54ab82bfb"}, + {file = "pillow-10.2.0-cp312-cp312-win32.whl", hash = "sha256:257d8788df5ca62c980314053197f4d46eefedf4e6175bc9412f14412ec4ea2f"}, + {file = "pillow-10.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:154e939c5f0053a383de4fd3d3da48d9427a7e985f58af8e94d0b3c9fcfcf4f9"}, + {file = "pillow-10.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8373c6c251f7ef8bda6675dd6d2b3a0fcc31edf1201266b5cf608b62a37407f9"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:870ea1ada0899fd0b79643990809323b389d4d1d46c192f97342eeb6ee0b8483"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b6b1e20608493548b1f32bce8cca185bf0480983890403d3b8753e44077129"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3031709084b6e7852d00479fd1d310b07d0ba82765f973b543c8af5061cf990e"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:3ff074fc97dd4e80543a3e91f69d58889baf2002b6be64347ea8cf5533188213"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:cb4c38abeef13c61d6916f264d4845fab99d7b711be96c326b84df9e3e0ff62d"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b1b3020d90c2d8e1dae29cf3ce54f8094f7938460fb5ce8bc5c01450b01fbaf6"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:170aeb00224ab3dc54230c797f8404507240dd868cf52066f66a41b33169bdbe"}, + {file = "pillow-10.2.0-cp38-cp38-win32.whl", hash = "sha256:c4225f5220f46b2fde568c74fca27ae9771536c2e29d7c04f4fb62c83275ac4e"}, + {file = "pillow-10.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0689b5a8c5288bc0504d9fcee48f61a6a586b9b98514d7d29b840143d6734f39"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:b792a349405fbc0163190fde0dc7b3fef3c9268292586cf5645598b48e63dc67"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c570f24be1e468e3f0ce7ef56a89a60f0e05b30a3669a459e419c6eac2c35364"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8ecd059fdaf60c1963c58ceb8997b32e9dc1b911f5da5307aab614f1ce5c2fb"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c365fd1703040de1ec284b176d6af5abe21b427cb3a5ff68e0759e1e313a5e7e"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:70c61d4c475835a19b3a5aa42492409878bbca7438554a1f89d20d58a7c75c01"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b6f491cdf80ae540738859d9766783e3b3c8e5bd37f5dfa0b76abdecc5081f13"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d189550615b4948f45252d7f005e53c2040cea1af5b60d6f79491a6e147eef7"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:49d9ba1ed0ef3e061088cd1e7538a0759aab559e2e0a80a36f9fd9d8c0c21591"}, + {file = "pillow-10.2.0-cp39-cp39-win32.whl", hash = "sha256:babf5acfede515f176833ed6028754cbcd0d206f7f614ea3447d67c33be12516"}, + {file = "pillow-10.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:0304004f8067386b477d20a518b50f3fa658a28d44e4116970abfcd94fac34a8"}, + {file = "pillow-10.2.0-cp39-cp39-win_arm64.whl", hash = "sha256:0fb3e7fc88a14eacd303e90481ad983fd5b69c761e9e6ef94c983f91025da869"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:322209c642aabdd6207517e9739c704dc9f9db943015535783239022002f054a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eedd52442c0a5ff4f887fab0c1c0bb164d8635b32c894bc1faf4c618dd89df2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb28c753fd5eb3dd859b4ee95de66cc62af91bcff5db5f2571d32a520baf1f04"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:33870dc4653c5017bf4c8873e5488d8f8d5f8935e2f1fb9a2208c47cdd66efd2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3c31822339516fb3c82d03f30e22b1d038da87ef27b6a78c9549888f8ceda39a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a2b56ba36e05f973d450582fb015594aaa78834fefe8dfb8fcd79b93e64ba4c6"}, + {file = "pillow-10.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d8e6aeb9201e655354b3ad049cb77d19813ad4ece0df1249d3c793de3774f8c7"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:2247178effb34a77c11c0e8ac355c7a741ceca0a732b27bf11e747bbc950722f"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15587643b9e5eb26c48e49a7b33659790d28f190fc514a322d55da2fb5c2950e"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753cd8f2086b2b80180d9b3010dd4ed147efc167c90d3bf593fe2af21265e5a5"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7c8f97e8e7a9009bcacbe3766a36175056c12f9a44e6e6f2d5caad06dcfbf03b"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d1b35bcd6c5543b9cb547dee3150c93008f8dd0f1fef78fc0cd2b141c5baf58a"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868"}, + {file = "pillow-10.2.0.tar.gz", hash = "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e"}, ] platformdirs = [ - {file = "platformdirs-3.8.0-py3-none-any.whl", hash = "sha256:ca9ed98ce73076ba72e092b23d3c93ea6c4e186b3f1c3dad6edd98ff6ffcca2e"}, - {file = "platformdirs-3.8.0.tar.gz", hash = "sha256:b0cabcb11063d21a0b261d557acb0a9d2126350e63b70cdf7db6347baea456dc"}, + {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, + {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, ] pluggy = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] pre-commit = [ {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"}, {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"}, ] prompt-toolkit = [ - {file = "prompt_toolkit-3.0.38-py3-none-any.whl", hash = "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f"}, - {file = "prompt_toolkit-3.0.38.tar.gz", hash = "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b"}, + {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, + {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, ] prov = [ {file = "prov-1.5.3-py2.py3-none-any.whl", hash = "sha256:b9c553cc0775bf193eb62de69e2a2b3bab545ab8e6bf597be243e8f0916e1d1b"}, @@ -2039,12 +2126,12 @@ py = [ {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] pygments = [ - {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, - {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, ] pyparsing = [ - {file = "pyparsing-3.1.0-py3-none-any.whl", hash = "sha256:d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1"}, - {file = "pyparsing-3.1.0.tar.gz", hash = "sha256:edb662d6fe322d6e990b1594b5feaeadf806803359e3d4d42f11e295e588f0ea"}, + {file = "pyparsing-3.1.1-py3-none-any.whl", hash = "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb"}, + {file = "pyparsing-3.1.1.tar.gz", hash = "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"}, ] pytest = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, @@ -2067,50 +2154,60 @@ python-dateutil = [ {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] pytz = [ - {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, - {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] pyyaml = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] rdflib = [ {file = "rdflib-6.3.2-py3-none-any.whl", hash = "sha256:36b4e74a32aa1e4fa7b8719876fb192f19ecd45ff932ea5ebbd2e417a0247e63"}, @@ -2125,27 +2222,32 @@ sal = [ {file = "sal-1.2.5.tar.gz", hash = "sha256:369f059fb775dbc320e380d590e04ac5692726f09bb4df08de35bfcad78f5f83"}, ] scikit-learn = [ - {file = "scikit-learn-1.3.0.tar.gz", hash = "sha256:8be549886f5eda46436b6e555b0e4873b4f10aa21c07df45c4bc1735afbccd7a"}, - {file = "scikit_learn-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:981287869e576d42c682cf7ca96af0c6ac544ed9316328fd0d9292795c742cf5"}, - {file = "scikit_learn-1.3.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:436aaaae2c916ad16631142488e4c82f4296af2404f480e031d866863425d2a2"}, - {file = "scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7e28d8fa47a0b30ae1bd7a079519dd852764e31708a7804da6cb6f8b36e3630"}, - {file = "scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae80c08834a473d08a204d966982a62e11c976228d306a2648c575e3ead12111"}, - {file = "scikit_learn-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:552fd1b6ee22900cf1780d7386a554bb96949e9a359999177cf30211e6b20df6"}, - {file = "scikit_learn-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79970a6d759eb00a62266a31e2637d07d2d28446fca8079cf9afa7c07b0427f8"}, - {file = "scikit_learn-1.3.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:850a00b559e636b23901aabbe79b73dc604b4e4248ba9e2d6e72f95063765603"}, - {file = "scikit_learn-1.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee04835fb016e8062ee9fe9074aef9b82e430504e420bff51e3e5fffe72750ca"}, - {file = "scikit_learn-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d953531f5d9f00c90c34fa3b7d7cfb43ecff4c605dac9e4255a20b114a27369"}, - {file = "scikit_learn-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:151ac2bf65ccf363664a689b8beafc9e6aae36263db114b4ca06fbbbf827444a"}, - {file = "scikit_learn-1.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a885a9edc9c0a341cab27ec4f8a6c58b35f3d449c9d2503a6fd23e06bbd4f6a"}, - {file = "scikit_learn-1.3.0-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:9877af9c6d1b15486e18a94101b742e9d0d2f343d35a634e337411ddb57783f3"}, - {file = "scikit_learn-1.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c470f53cea065ff3d588050955c492793bb50c19a92923490d18fcb637f6383a"}, - {file = "scikit_learn-1.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6e2d7389542eae01077a1ee0318c4fec20c66c957f45c7aac0c6eb0fe3c612"}, - {file = "scikit_learn-1.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:3a11936adbc379a6061ea32fa03338d4ca7248d86dd507c81e13af428a5bc1db"}, - {file = "scikit_learn-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:998d38fcec96584deee1e79cd127469b3ad6fefd1ea6c2dfc54e8db367eb396b"}, - {file = "scikit_learn-1.3.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ded35e810438a527e17623ac6deae3b360134345b7c598175ab7741720d7ffa7"}, - {file = "scikit_learn-1.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e8102d5036e28d08ab47166b48c8d5e5810704daecf3a476a4282d562be9a28"}, - {file = "scikit_learn-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7617164951c422747e7c32be4afa15d75ad8044f42e7d70d3e2e0429a50e6718"}, - {file = "scikit_learn-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:1d54fb9e6038284548072df22fd34777e434153f7ffac72c8596f2d6987110dd"}, + {file = "scikit-learn-1.3.2.tar.gz", hash = "sha256:a2f54c76accc15a34bfb9066e6c7a56c1e7235dda5762b990792330b52ccfb05"}, + {file = "scikit_learn-1.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e326c0eb5cf4d6ba40f93776a20e9a7a69524c4db0757e7ce24ba222471ee8a1"}, + {file = "scikit_learn-1.3.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:535805c2a01ccb40ca4ab7d081d771aea67e535153e35a1fd99418fcedd1648a"}, + {file = "scikit_learn-1.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1215e5e58e9880b554b01187b8c9390bf4dc4692eedeaf542d3273f4785e342c"}, + {file = "scikit_learn-1.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ee107923a623b9f517754ea2f69ea3b62fc898a3641766cb7deb2f2ce450161"}, + {file = "scikit_learn-1.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:35a22e8015048c628ad099da9df5ab3004cdbf81edc75b396fd0cff8699ac58c"}, + {file = "scikit_learn-1.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6fb6bc98f234fda43163ddbe36df8bcde1d13ee176c6dc9b92bb7d3fc842eb66"}, + {file = "scikit_learn-1.3.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:18424efee518a1cde7b0b53a422cde2f6625197de6af36da0b57ec502f126157"}, + {file = "scikit_learn-1.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3271552a5eb16f208a6f7f617b8cc6d1f137b52c8a1ef8edf547db0259b2c9fb"}, + {file = "scikit_learn-1.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4144a5004a676d5022b798d9e573b05139e77f271253a4703eed295bde0433"}, + {file = "scikit_learn-1.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:67f37d708f042a9b8d59551cf94d30431e01374e00dc2645fa186059c6c5d78b"}, + {file = "scikit_learn-1.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8db94cd8a2e038b37a80a04df8783e09caac77cbe052146432e67800e430c028"}, + {file = "scikit_learn-1.3.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:61a6efd384258789aa89415a410dcdb39a50e19d3d8410bd29be365bcdd512d5"}, + {file = "scikit_learn-1.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb06f8dce3f5ddc5dee1715a9b9f19f20d295bed8e3cd4fa51e1d050347de525"}, + {file = "scikit_learn-1.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b2de18d86f630d68fe1f87af690d451388bb186480afc719e5f770590c2ef6c"}, + {file = "scikit_learn-1.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:0402638c9a7c219ee52c94cbebc8fcb5eb9fe9c773717965c1f4185588ad3107"}, + {file = "scikit_learn-1.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a19f90f95ba93c1a7f7924906d0576a84da7f3b2282ac3bfb7a08a32801add93"}, + {file = "scikit_learn-1.3.2-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:b8692e395a03a60cd927125eef3a8e3424d86dde9b2370d544f0ea35f78a8073"}, + {file = "scikit_learn-1.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15e1e94cc23d04d39da797ee34236ce2375ddea158b10bee3c343647d615581d"}, + {file = "scikit_learn-1.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:785a2213086b7b1abf037aeadbbd6d67159feb3e30263434139c98425e3dcfcf"}, + {file = "scikit_learn-1.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:64381066f8aa63c2710e6b56edc9f0894cc7bf59bd71b8ce5613a4559b6145e0"}, + {file = "scikit_learn-1.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6c43290337f7a4b969d207e620658372ba3c1ffb611f8bc2b6f031dc5c6d1d03"}, + {file = "scikit_learn-1.3.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:dc9002fc200bed597d5d34e90c752b74df516d592db162f756cc52836b38fe0e"}, + {file = "scikit_learn-1.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d08ada33e955c54355d909b9c06a4789a729977f165b8bae6f225ff0a60ec4a"}, + {file = "scikit_learn-1.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:763f0ae4b79b0ff9cca0bf3716bcc9915bdacff3cebea15ec79652d1cc4fa5c9"}, + {file = "scikit_learn-1.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:ed932ea780517b00dae7431e031faae6b49b20eb6950918eb83bd043237950e0"}, ] scipy = [ {file = "scipy-1.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7354fd7527a4b0377ce55f286805b34e8c54b91be865bac273f527e1b839019"}, @@ -2170,10 +2272,6 @@ scipy = [ {file = "scipy-1.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:7ff7f37b1bf4417baca958d254e8e2875d0cc23aaadbe65b3d5b3077b0eb23ea"}, {file = "scipy-1.10.1.tar.gz", hash = "sha256:2cf9dfb80a7b4589ba4c40ce7588986d6d5cebc5457cad2c2880f6bc2d42f3a5"}, ] -setuptools-scm = [ - {file = "setuptools_scm-7.1.0-py3-none-any.whl", hash = "sha256:73988b6d848709e2af142aa48c986ea29592bbcfca5375678064708205253d8e"}, - {file = "setuptools_scm-7.1.0.tar.gz", hash = "sha256:6c508345a771aad7d56ebff0e70628bf2b0ec7573762be9960214730de278f27"}, -] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -2183,12 +2281,12 @@ sortedcontainers = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] stack-data = [ - {file = "stack_data-0.6.2-py3-none-any.whl", hash = "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8"}, - {file = "stack_data-0.6.2.tar.gz", hash = "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815"}, + {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, + {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, ] threadpoolctl = [ - {file = "threadpoolctl-3.1.0-py3-none-any.whl", hash = "sha256:8b99adda265feb6773280df41eece7b2e6561b772d21ffd52e372f999024907b"}, - {file = "threadpoolctl-3.1.0.tar.gz", hash = "sha256:a335baacfaa4400ae1f0d8e3a58d6674d2f8828e3716bb2802c44955ad391380"}, + {file = "threadpoolctl-3.2.0-py3-none-any.whl", hash = "sha256:2b7818516e423bdaebb97c723f86a7c6b0a83d3f3b0970328d66f4d9104dc032"}, + {file = "threadpoolctl-3.2.0.tar.gz", hash = "sha256:c96a0ba3bdddeaca37dc4cc7344aafad41cdb8c313f74fdfe387a867bba93355"}, ] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, @@ -2199,42 +2297,42 @@ tomli = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] tqdm = [ - {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, - {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, + {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, + {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, ] traitlets = [ - {file = "traitlets-5.9.0-py3-none-any.whl", hash = "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8"}, - {file = "traitlets-5.9.0.tar.gz", hash = "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"}, + {file = "traitlets-5.14.1-py3-none-any.whl", hash = "sha256:2e5a030e6eff91737c643231bfcf04a65b0132078dad75e4936700b213652e74"}, + {file = "traitlets-5.14.1.tar.gz", hash = "sha256:8585105b371a04b8316a43d5ce29c098575c2e477850b62b848b964f1444527e"}, ] types-setuptools = [ {file = "types-setuptools-57.4.18.tar.gz", hash = "sha256:8ee03d823fe7fda0bd35faeae33d35cb5c25b497263e6a58b34c4cfd05f40bcf"}, {file = "types_setuptools-57.4.18-py3-none-any.whl", hash = "sha256:9660b8774b12cd61b448e2fd87a667c02e7ec13ce9f15171f1d49a4654c4df6a"}, ] typing-extensions = [ - {file = "typing_extensions-4.7.0-py3-none-any.whl", hash = "sha256:5d8c9dac95c27d20df12fb1d97b9793ab8b2af8a3a525e68c80e21060c161771"}, - {file = "typing_extensions-4.7.0.tar.gz", hash = "sha256:935ccf31549830cda708b42289d44b6f74084d616a00be651601a4f968e77c82"}, + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] urllib3 = [ - {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, - {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, + {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, + {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, ] virtualenv = [ - {file = "virtualenv-20.23.1-py3-none-any.whl", hash = "sha256:34da10f14fea9be20e0fd7f04aba9732f84e593dac291b757ce42e3368a39419"}, - {file = "virtualenv-20.23.1.tar.gz", hash = "sha256:8ff19a38c1021c742148edc4f81cb43d7f8c6816d2ede2ab72af5b84c749ade1"}, + {file = "virtualenv-20.25.0-py3-none-any.whl", hash = "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3"}, + {file = "virtualenv-20.25.0.tar.gz", hash = "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"}, ] wcwidth = [ - {file = "wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"}, - {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, ] werkzeug = [ - {file = "Werkzeug-2.3.6-py3-none-any.whl", hash = "sha256:935539fa1413afbb9195b24880778422ed620c0fc09670945185cce4d91a8890"}, - {file = "Werkzeug-2.3.6.tar.gz", hash = "sha256:98c774df2f91b05550078891dee5f0eb0cb797a522c757a2452b9cee5b202330"}, + {file = "werkzeug-3.0.1-py3-none-any.whl", hash = "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10"}, + {file = "werkzeug-3.0.1.tar.gz", hash = "sha256:507e811ecea72b18a404947aded4b3390e1db8f826b494d76550ef45bb3b1dcc"}, ] xarray = [ {file = "xarray-0.16.2-py3-none-any.whl", hash = "sha256:88d0f80145999443730f87c7575e426d377f0b62677d915f21059cc364b4a1e1"}, {file = "xarray-0.16.2.tar.gz", hash = "sha256:38e8439d6c91bcd5b7c0fca349daf8e0643ac68850c987262d53526e9d7d01e4"}, ] zipp = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, + {file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"}, + {file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"}, ] From b006ca56806fb395e511976dc5983268169b2bed Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 17:08:40 +0000 Subject: [PATCH 268/288] python >= 3.9 required --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c455fda5..20990ce8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ include = [".git", "indica/git_version"] [tool.poetry.dependencies] corner = "^1.0" pandas = "1.3.0" -python = ">=3.8,<3.11" +python = ">=3.9,<3.11" prov = "^1.5.3" netCDF4 = "^1.5.4" numpy = "1.21.0" @@ -55,6 +55,7 @@ emcee = "^3.1.4" flatdict = "^4.0.1" tqdm = "^4.65.0" scikit-learn = "^1.3.0" +Pint = "^0.23" [tool.poetry.dev-dependencies] click = "^8.0.0" From a2876ae8b1f68db96695307117cb1279657826b6 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Thu, 18 Jan 2024 17:09:58 +0000 Subject: [PATCH 269/288] pint added for standard_utility --- poetry.lock | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index e7d0ab06..fb5cb1fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -696,6 +696,29 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa typing = ["typing-extensions"] xmp = ["defusedxml"] +[[package]] +name = "pint" +version = "0.23" +description = "Physical quantities module" +category = "main" +optional = false +python-versions = ">=3.9" + +[package.dependencies] +typing-extensions = "*" + +[package.extras] +babel = ["babel (<=2.8)"] +bench = ["pytest", "pytest-codspeed"] +dask = ["dask"] +mip = ["mip (>=1.13)"] +numpy = ["numpy (>=1.19.5)"] +pandas = ["pint-pandas (>=0.3)"] +test = ["pytest", "pytest-benchmark", "pytest-cov", "pytest-mpl", "pytest-subtests"] +testbase = ["pytest", "pytest-benchmark", "pytest-cov", "pytest-subtests"] +uncertainties = ["uncertainties (>=3.1.6)"] +xarray = ["xarray"] + [[package]] name = "platformdirs" version = "4.1.0" @@ -1185,8 +1208,8 @@ testing = ["big-o", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "1.1" -python-versions = ">=3.8,<3.11" -content-hash = "27cf16cddd6c0208dbd532d34116d912270318f680df17d1026f89a9fabb4500" +python-versions = ">=3.9,<3.11" +content-hash = "359cc0bf9eca5bb1c8601d3c6e1b55b4cba1adac34214d0c9819a527bc661dc9" [metadata.files] aniso8601 = [ @@ -2093,6 +2116,10 @@ pillow = [ {file = "pillow-10.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868"}, {file = "pillow-10.2.0.tar.gz", hash = "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e"}, ] +pint = [ + {file = "Pint-0.23-py3-none-any.whl", hash = "sha256:df79b6b5f1beb7ed0cd55d91a0766fc55f972f757a9364e844958c05e8eb66f9"}, + {file = "Pint-0.23.tar.gz", hash = "sha256:e1509b91606dbc52527c600a4ef74ffac12fff70688aff20e9072409346ec9b4"}, +] platformdirs = [ {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, From 5663c3530e532666174ca5d5f929f50b61a78ed9 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 Jan 2024 09:30:50 +0000 Subject: [PATCH 270/288] default to 43000000 --- indica/writers/bda_tree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index b26f4a4d..9535af9b 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -202,7 +202,7 @@ def bda(): def create_nodes( - pulse_to_write=23000101, + pulse_to_write=43000000, run="RUN01", best=True, diagnostic_quantities=DIAGNOSTIC_QUANTITY, @@ -322,4 +322,4 @@ def query_yes_no( if __name__ == "__main__": - create_nodes(pulse_to_write=23000101, mode="EDIT", run="RUN02") + create_nodes(pulse_to_write=43000000, mode="EDIT", run="RUN01") From ed43c55e963d01a41621e3427f162278cd8c1b06 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 Jan 2024 09:32:42 +0000 Subject: [PATCH 271/288] bda_nodes function is now a global var --- indica/writers/bda_tree.py | 332 ++++++++++++++++++------------------- 1 file changed, 165 insertions(+), 167 deletions(-) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 9535af9b..243a8dd0 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -7,191 +7,189 @@ import standard_utility as util -def bda(): - nodes = { - "TIME": ("NUMERIC", "time vector of optimisation, s"), - "INPUT": { - "BURN_FRAC": ("NUMERIC", "Burn in fraction for chains"), - "ITER": ("NUMERIC", "Maximum iterations of optimiser"), - "PARAM_NAMES": ("TEXT", "Names of parameters optimised"), - "OPT_QUANTITY": ("TEXT", "Names of quantities optimised"), - "MODEL_KWARGS": ("TEXT", "Model key word arguments"), - # "OPT_KWARGS": ("TEXT", "Optimiser key word arguments"), - "PULSE": ("NUMERIC", "Pulse number"), - "TSTART": ("NUMERIC", "Start of time vector, s"), - "TEND": ("NUMERIC", "End of time vector, s"), - "DT": ("NUMERIC", "Distance between time points, s"), - "IMPURITIES": ("TEXT", "Names of impurity elements"), - "MAIN_ION": ("TEXT", "Name of main ion"), - }, - "METADATA": { - "GITCOMMIT": ("TEXT", "Commit ID used for run"), - "USER": ("TEXT", "Username of owner"), - "EQUIL": ("TEXT", "Equilibrium used"), - }, - "PROFILES": { - "PSI_NORM": { - "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), - "RHOT": ("SIGNAL", "Radial vector, toroidal flux"), - "NE": ("SIGNAL", "Electron density, m^-3"), - "NI": ("SIGNAL", "Ion density, m^-3"), - "TE": ("SIGNAL", "Electron temperature, eV"), - "TI": ("SIGNAL", "Ion temperature of main ion, eV"), - "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1, eV"), - "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), - "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), - "NIZ1": ("SIGNAL", "Density of impurity Z1, m^-3"), - "NIZ2": ("SIGNAL", "Density of impurity Z2, m^-3"), - "NIZ3": ("SIGNAL", "Density of impurity Z3, m^-3"), - "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), - "NFAST": ("SIGNAL", "Density of fast ion, m^-3"), - "ZI": ("SIGNAL", "Average charge of main ion, "), - "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), - "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), - "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), - "ZEFF": ("SIGNAL", "Effective charge, "), - "P": ("SIGNAL", "Pressure,Pa"), - "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), - "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), - "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), - "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), - "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), - "TIZ1_ERR": ("SIGNAL", "Ion temperature of impurity Z1 error, eV"), - "TIZ2_ERR": ("SIGNAL", "Ion temperature of impurity Z2 error, eV"), - "TIZ3_ERR": ("SIGNAL", "Ion temperature of impurity Z3 error, eV"), - "NIZ1_ERR": ("SIGNAL", "Density of impurity Z1 error, m^-3"), - "NIZ2_ERR": ("SIGNAL", "Density of impurity Z2 error, m^-3"), - "NIZ3_ERR": ("SIGNAL", "Density of impurity Z3 error, m^-3"), - "NNEUTR_ERR": ("SIGNAL", "Density of neutral main ion error, m^-3"), - "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), - "ZI_ERR": ("SIGNAL", "Average charge of main ion error, "), - "ZIM1_ERR": ("SIGNAL", "Average charge of impurity IMP1 error, "), - "ZIM2_ERR": ("SIGNAL", "Average charge of impurity IMP2 error, "), - "ZIM3_ERR": ("SIGNAL", "Average charge of impurity IMP3 error, "), - "ZEFF_ERR": ("SIGNAL", "Effective charge error, "), - }, - - "R_MIDPLANE": { - "RPOS": ("NUMERIC", "Major radius position of measurement, m"), - "ZPOS": ("NUMERIC", "Z position of measurement, m"), - "NE": ("SIGNAL", "Electron density, m^-3"), - "NI": ("SIGNAL", "Ion density, m^-3"), - "TE": ("SIGNAL", "Electron temperature, eV"), - "TI": ("SIGNAL", "Ion temperature of main ion, eV"), - "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1, eV"), - "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), - "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), - "NIZ1": ("SIGNAL", "Density of impurity Z1, m^-3"), - "NIZ2": ("SIGNAL", "Density of impurity Z2, m^-3"), - "NIZ3": ("SIGNAL", "Density of impurity Z3, m^-3"), - "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), - "NFAST": ("SIGNAL", "Density of fast ion, m^-3"), - "ZI": ("SIGNAL", "Average charge of main ion, "), - "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), - "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), - "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), - "ZEFF": ("SIGNAL", "Effective charge, "), - "P": ("SIGNAL", "Pressure,Pa"), - "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), - "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), - "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), - "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), - "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), - "TIZ1_ERR": ("SIGNAL", "Ion temperature of impurity Z1 error, eV"), - "TIZ2_ERR": ("SIGNAL", "Ion temperature of impurity Z2 error, eV"), - "TIZ3_ERR": ("SIGNAL", "Ion temperature of impurity Z3 error, eV"), - "NIZ1_ERR": ("SIGNAL", "Density of impurity Z1 error, m^-3"), - "NIZ2_ERR": ("SIGNAL", "Density of impurity Z2 error, m^-3"), - "NIZ3_ERR": ("SIGNAL", "Density of impurity Z3 error, m^-3"), - "NNEUTR_ERR": ("SIGNAL", "Density of neutral main ion error, m^-3"), - "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), - "ZI_ERR": ("SIGNAL", "Average charge of main ion error, "), - "ZIM1_ERR": ("SIGNAL", "Average charge of impurity IMP1 error, "), - "ZIM2_ERR": ("SIGNAL", "Average charge of impurity IMP2 error, "), - "ZIM3_ERR": ("SIGNAL", "Average charge of impurity IMP3 error, "), - "ZEFF_ERR": ("SIGNAL", "Effective charge error, "), - }, - }, - - "PROFILE_STAT": { - "SAMPLE_IDX": ("NUMERIC", "Index of the optimisation samples"), +BDA_NODES = { + "TIME": ("NUMERIC", "time vector of optimisation, s"), + "INPUT": { + "BURN_FRAC": ("NUMERIC", "Burn in fraction for chains"), + "ITER": ("NUMERIC", "Maximum iterations of optimiser"), + "PARAM_NAMES": ("TEXT", "Names of parameters optimised"), + "OPT_QUANTITY": ("TEXT", "Names of quantities optimised"), + "MODEL_KWARGS": ("TEXT", "Model key word arguments"), + # "OPT_KWARGS": ("TEXT", "Optimiser key word arguments"), + "PULSE": ("NUMERIC", "Pulse number"), + "TSTART": ("NUMERIC", "Start of time vector, s"), + "TEND": ("NUMERIC", "End of time vector, s"), + "DT": ("NUMERIC", "Distance between time points, s"), + "IMPURITIES": ("TEXT", "Names of impurity elements"), + "MAIN_ION": ("TEXT", "Name of main ion"), + }, + "METADATA": { + "GITCOMMIT": ("TEXT", "Commit ID used for run"), + "USER": ("TEXT", "Username of owner"), + "EQUIL": ("TEXT", "Equilibrium used"), + }, + "PROFILES": { + "PSI_NORM": { "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "RHOT": ("SIGNAL", "Radial vector, toroidal flux"), "NE": ("SIGNAL", "Electron density, m^-3"), "NI": ("SIGNAL", "Ion density, m^-3"), "TE": ("SIGNAL", "Electron temperature, eV"), "TI": ("SIGNAL", "Ion temperature of main ion, eV"), - "TIZ1": ("SIGNAL", "Ion temperature of impurity IMP1, eV"), - "TIZ2": ("SIGNAL", "Ion temperature of impurity IMP2, eV"), - "TIZ3": ("SIGNAL", "Ion temperature of impurity IMP3, eV"), - "NIZ1": ("SIGNAL", "Density of impurity IMP1, m^-3"), - "NIZ2": ("SIGNAL", "Density of impurity IMP2, m^-3"), - "NIZ3": ("SIGNAL", "Density of impurity IMP3, m^-3"), + "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1, eV"), + "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), + "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), + "NIZ1": ("SIGNAL", "Density of impurity Z1, m^-3"), + "NIZ2": ("SIGNAL", "Density of impurity Z2, m^-3"), + "NIZ3": ("SIGNAL", "Density of impurity Z3, m^-3"), "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), - "NFAST": ("SIGNAL", "Density of fast ions, m^-3"), + "NFAST": ("SIGNAL", "Density of fast ion, m^-3"), "ZI": ("SIGNAL", "Average charge of main ion, "), "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), "ZEFF": ("SIGNAL", "Effective charge, "), "P": ("SIGNAL", "Pressure,Pa"), - "VOLUME": ("SIGNAL", "Volume inside magnetic surface, m^3"), - }, - "GLOBAL": { - "NE0": ("SIGNAL", "Central electron density, m^-3"), - "NI0": ("SIGNAL", "Central ion density, m^-3"), - "TE0": ("SIGNAL", "Central electron temperature, eV"), - "TI0": ("SIGNAL", "Central ion temperature of main ion, eV"), - "TI0Z1": ("SIGNAL", "Central ion temperature of impurity Z1, eV"), - "TI0Z2": ("SIGNAL", "Central ion temperature of impurity Z2, eV"), - "TI0Z3": ("SIGNAL", "Central ion temperature of impurity Z3, eV"), - "NI0Z1": ("SIGNAL", "Central density of impurity Z1, m^-3"), - "NI0Z2": ("SIGNAL", "Central density of impurity Z2, m^-3"), - "NI0Z3": ("SIGNAL", "Central density of impurity Z3, m^-3"), - "NE0_ERR": ("SIGNAL", "Central electron density error, m^-3"), - "NI0_ERR": ("SIGNAL", "Central ion density error, m^-3"), - "TE0_ERR": ("SIGNAL", "Central electron temperature error, eV"), - "TI0_ERR": ("SIGNAL", "Central ion temperature of main ion error, eV"), - "TI0Z1_ERR": ("SIGNAL", "Central ion temperature of impurity Z1 error, eV"), - "TI0Z2_ERR": ("SIGNAL", "Central ion temperature of impurity Z2 error, eV"), - "TI0Z3_ERR": ("SIGNAL", "Central ion temperature of impurity Z3 error, eV"), - "NI0Z1_ERR": ("SIGNAL", "Central density of impurity Z1 error, m^-3"), - "NI0Z2_ERR": ("SIGNAL", "Central density of impurity Z2 error, m^-3"), - "NI0Z3_ERR": ("SIGNAL", "Central density of impurity Z3 error, m^-3"), - "WP": ("SIGNAL", "Stored energy, J"), - "WTH": ("SIGNAL", "Thermal component of stored energy, J"), - "PTOT": ("SIGNAL", "Total pressure, Pa"), - "PTH": ("SIGNAL", "Thermal pressure, Pa"), - "WP_ERR": ("SIGNAL", "Stored energy error, J"), - "WTH_ERR": ("SIGNAL", "Thermal component of stored energy error, J"), - "PTOT_ERR": ("SIGNAL", "Total pressure error, Pa"), - "PTH_ERR": ("SIGNAL", "Thermal pressure error, Pa"), - }, - "PHANTOMS": { - "FLAG": ("TEXT", "True if phantoms used"), - "RHO_POLOIDAL": ( - "NUMERIC", - "Radial vector, Sqrt of normalised poloidal flux", - ), + "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), + "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), + "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), + "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), + "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), + "TIZ1_ERR": ("SIGNAL", "Ion temperature of impurity Z1 error, eV"), + "TIZ2_ERR": ("SIGNAL", "Ion temperature of impurity Z2 error, eV"), + "TIZ3_ERR": ("SIGNAL", "Ion temperature of impurity Z3 error, eV"), + "NIZ1_ERR": ("SIGNAL", "Density of impurity Z1 error, m^-3"), + "NIZ2_ERR": ("SIGNAL", "Density of impurity Z2 error, m^-3"), + "NIZ3_ERR": ("SIGNAL", "Density of impurity Z3 error, m^-3"), + "NNEUTR_ERR": ("SIGNAL", "Density of neutral main ion error, m^-3"), + "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), + "ZI_ERR": ("SIGNAL", "Average charge of main ion error, "), + "ZIM1_ERR": ("SIGNAL", "Average charge of impurity IMP1 error, "), + "ZIM2_ERR": ("SIGNAL", "Average charge of impurity IMP2 error, "), + "ZIM3_ERR": ("SIGNAL", "Averagnodese charge of impurity IMP3 error, "), + "ZEFF_ERR": ("SIGNAL", "Effective charge error, "), + }, + + "R_MIDPLANE": { + "RPOS": ("NUMERIC", "Major radius position of measurement, m"), + "ZPOS": ("NUMERIC", "Z position of measurement, m"), "NE": ("SIGNAL", "Electron density, m^-3"), "NI": ("SIGNAL", "Ion density, m^-3"), "TE": ("SIGNAL", "Electron temperature, eV"), "TI": ("SIGNAL", "Ion temperature of main ion, eV"), - "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1 , eV"), + "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1, eV"), "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), - "NIZ1": ("SIGNAL", "Impurity density of Z1, m^-3"), - "NIZ2": ("SIGNAL", "Impurity density of Z2, m^-3"), - "NIZ3": ("SIGNAL", "Impurity density of Z3, m^-3"), - }, - "OPTIMISATION": { - "ACCEPT_FRAC": ("NUMERIC", "Fraction of samples accepted by optimiser"), - "AUTO_CORR": ("NUMERIC", "Auto-correlation"), - "POST_SAMPLE": ("NUMERIC", "Posterior probability samples"), - "PRIOR_SAMPLE": ("NUMERIC", "Prior samples"), - "GELMAN_RUBIN": ("NUMERIC", "Gelmin-Rubin convergence diagnostic"), + "NIZ1": ("SIGNAL", "Density of impurity Z1, m^-3"), + "NIZ2": ("SIGNAL", "Density of impurity Z2, m^-3"), + "NIZ3": ("SIGNAL", "Density of impurity Z3, m^-3"), + "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), + "NFAST": ("SIGNAL", "Density of fast ion, m^-3"), + "ZI": ("SIGNAL", "Average charge of main ion, "), + "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), + "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), + "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), + "ZEFF": ("SIGNAL", "Effective charge, "), + "P": ("SIGNAL", "Pressure,Pa"), + "VOLUME": ("SIGNAL", "Volume inside magnetic surface,m^3"), + "NE_ERR": ("SIGNAL", "Electron density error, m^-3"), + "NI_ERR": ("SIGNAL", "Ion density error, m^-3"), + "TE_ERR": ("SIGNAL", "Electron temperature error, eV"), + "TI_ERR": ("SIGNAL", "Ion temperature of main ion error, eV"), + "TIZ1_ERR": ("SIGNAL", "Ion temperature of impurity Z1 error, eV"), + "TIZ2_ERR": ("SIGNAL", "Ion temperature of impurity Z2 error, eV"), + "TIZ3_ERR": ("SIGNAL", "Ion temperature of impurity Z3 error, eV"), + "NIZ1_ERR": ("SIGNAL", "Density of impurity Z1 error, m^-3"), + "NIZ2_ERR": ("SIGNAL", "Density of impurity Z2 error, m^-3"), + "NIZ3_ERR": ("SIGNAL", "Density of impurity Z3 error, m^-3"), + "NNEUTR_ERR": ("SIGNAL", "Density of neutral main ion error, m^-3"), + "NFAST_ERR": ("SIGNAL", "Density of fast ion error, m^-3"), + "ZI_ERR": ("SIGNAL", "Average charge of main ion error, "), + "ZIM1_ERR": ("SIGNAL", "Average charge of impurity IMP1 error, "), + "ZIM2_ERR": ("SIGNAL", "Average charge of impurity IMP2 error, "), + "ZIM3_ERR": ("SIGNAL", "Average charge of impurity IMP3 error, "), + "ZEFF_ERR": ("SIGNAL", "Effective charge error, "), }, - } - return nodes + }, + + "PROFILE_STAT": { + "SAMPLE_IDX": ("NUMERIC", "Index of the optimisation samples"), + "RHOP": ("NUMERIC", "Radial vector, Sqrt of normalised poloidal flux"), + "NE": ("SIGNAL", "Electron density, m^-3"), + "NI": ("SIGNAL", "Ion density, m^-3"), + "TE": ("SIGNAL", "Electron temperature, eV"), + "TI": ("SIGNAL", "Ion temperature of main ion, eV"), + "TIZ1": ("SIGNAL", "Ion temperature of impurity IMP1, eV"), + "TIZ2": ("SIGNAL", "Ion temperature of impurity IMP2, eV"), + "TIZ3": ("SIGNAL", "Ion temperature of impurity IMP3, eV"), + "NIZ1": ("SIGNAL", "Density of impurity IMP1, m^-3"), + "NIZ2": ("SIGNAL", "Density of impurity IMP2, m^-3"), + "NIZ3": ("SIGNAL", "Density of impurity IMP3, m^-3"), + "NNEUTR": ("SIGNAL", "Density of neutral main ion, m^-3"), + "NFAST": ("SIGNAL", "Density of fast ions, m^-3"), + "ZI": ("SIGNAL", "Average charge of main ion, "), + "ZIM1": ("SIGNAL", "Average charge of impurity IMP1, "), + "ZIM2": ("SIGNAL", "Average charge of impurity IMP2, "), + "ZIM3": ("SIGNAL", "Average charge of impurity IMP3, "), + "ZEFF": ("SIGNAL", "Effective charge, "), + "P": ("SIGNAL", "Pressure,Pa"), + "VOLUME": ("SIGNAL", "Volume inside magnetic surface, m^3"), + }, + "GLOBAL": { + "NE0": ("SIGNAL", "Central electron density, m^-3"), + "NI0": ("SIGNAL", "Central ion density, m^-3"), + "TE0": ("SIGNAL", "Central electron temperature, eV"), + "TI0": ("SIGNAL", "Central ion temperature of main ion, eV"), + "TI0Z1": ("SIGNAL", "Central ion temperature of impurity Z1, eV"), + "TI0Z2": ("SIGNAL", "Central ion temperature of impurity Z2, eV"), + "TI0Z3": ("SIGNAL", "Central ion temperature of impurity Z3, eV"), + "NI0Z1": ("SIGNAL", "Central density of impurity Z1, m^-3"), + "NI0Z2": ("SIGNAL", "Central density of impurity Z2, m^-3"), + "NI0Z3": ("SIGNAL", "Central density of impurity Z3, m^-3"), + "NE0_ERR": ("SIGNAL", "Central electron density error, m^-3"), + "NI0_ERR": ("SIGNAL", "Central ion density error, m^-3"), + "TE0_ERR": ("SIGNAL", "Central electron temperature error, eV"), + "TI0_ERR": ("SIGNAL", "Central ion temperature of main ion error, eV"), + "TI0Z1_ERR": ("SIGNAL", "Central ion temperature of impurity Z1 error, eV"), + "TI0Z2_ERR": ("SIGNAL", "Central ion temperature of impurity Z2 error, eV"), + "TI0Z3_ERR": ("SIGNAL", "Central ion temperature of impurity Z3 error, eV"), + "NI0Z1_ERR": ("SIGNAL", "Central density of impurity Z1 error, m^-3"), + "NI0Z2_ERR": ("SIGNAL", "Central density of impurity Z2 error, m^-3"), + "NI0Z3_ERR": ("SIGNAL", "Central density of impurity Z3 error, m^-3"), + "WP": ("SIGNAL", "Stored energy, J"), + "WTH": ("SIGNAL", "Thermal component of stored energy, J"), + "PTOT": ("SIGNAL", "Total pressure, Pa"), + "PTH": ("SIGNAL", "Thermal pressure, Pa"), + "WP_ERR": ("SIGNAL", "Stored energy error, J"), + "WTH_ERR": ("SIGNAL", "Thermal component of stored energy error, J"), + "PTOT_ERR": ("SIGNAL", "Total pressure error, Pa"), + "PTH_ERR": ("SIGNAL", "Thermal pressure error, Pa"), + }, + "PHANTOMS": { + "FLAG": ("TEXT", "True if phantoms used"), + "RHO_POLOIDAL": ( + "NUMERIC", + "Radial vector, Sqrt of normalised poloidal flux", + ), + "NE": ("SIGNAL", "Electron density, m^-3"), + "NI": ("SIGNAL", "Ion density, m^-3"), + "TE": ("SIGNAL", "Electron temperature, eV"), + "TI": ("SIGNAL", "Ion temperature of main ion, eV"), + "TIZ1": ("SIGNAL", "Ion temperature of impurity Z1 , eV"), + "TIZ2": ("SIGNAL", "Ion temperature of impurity Z2, eV"), + "TIZ3": ("SIGNAL", "Ion temperature of impurity Z3, eV"), + "NIZ1": ("SIGNAL", "Impurity density of Z1, m^-3"), + "NIZ2": ("SIGNAL", "Impurity density of Z2, m^-3"), + "NIZ3": ("SIGNAL", "Impurity density of Z3, m^-3"), + }, + "OPTIMISATION": { + "ACCEPT_FRAC": ("NUMERIC", "Fraction of samples accepted by optimiser"), + "AUTO_CORR": ("NUMERIC", "Auto-correlation"), + "POST_SAMPLE": ("NUMERIC", "Posterior probability samples"), + "PRIOR_SAMPLE": ("NUMERIC", "Prior samples"), + "GELMAN_RUBIN": ("NUMERIC", "Gelmin-Rubin convergence diagnostic"), + }, +} DIAGNOSTIC_QUANTITY = [ @@ -208,7 +206,7 @@ def create_nodes( diagnostic_quantities=DIAGNOSTIC_QUANTITY, mode="EDIT", ): - bda_nodes = bda() + bda_nodes = BDA_NODES quant_list = [ item.upper().split(".") for item in diagnostic_quantities ] # replace OPTIMISED_QUANTITY From 865c4a1435d582e17e8452696199d32b92c98222 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 Jan 2024 09:48:04 +0000 Subject: [PATCH 272/288] check for tree and use NEW or EDIT mode --- indica/workflows/bayes_workflow.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index eae6ed2b..68cacce7 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -29,7 +29,7 @@ from indica.operators.gpr_fit import post_process_ts, gpr_fit_ts from indica.workflows.abstract_bayes_workflow import AbstractBayesWorkflow from indica.workflows.bayes_plots import plot_bayes_result -from indica.writers.bda_tree import create_nodes +from indica.writers.bda_tree import create_nodes, does_tree_exist from indica.writers.bda_tree import write_nodes from indica.equilibrium import Equilibrium from indica.equilibrium import fake_equilibrium @@ -401,7 +401,8 @@ def fit_ts_profile(self, data_context, split="LFS", quant="ne"): length_scale_bounds=(0.4, 0.7)) + kernels.WhiteKernel( noise_level_bounds=(0.01, 10)) - processed_data = post_process_ts(deepcopy(data_context.binned_data), data_context.equilibrium, quant, data_context.pulse, split=split, ) + processed_data = post_process_ts(deepcopy(data_context.binned_data), data_context.equilibrium, quant, + data_context.pulse, split=split, ) fit, _ = gpr_fit_ts(data=processed_data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) fit = xr.where(fit < 0, 1e-10, fit) return fit @@ -443,6 +444,7 @@ def map_profiles_to_midplane(self, blobs): midplane_profiles[profile] = midplane_profiles[profile].swap_dims({"channel": "R"}) return midplane_profiles + @dataclass class ModelSettings: init_kwargs: dict = field(default_factory=lambda: { @@ -699,7 +701,8 @@ class MockData(PhantomData): "cxff_pi": pi_transform_example(5), "cxff_tws_c": pi_transform_example(3), "ts": ts_transform_example(11), - "efit": lambda: None, # placeholder to stop missing_transforms error + "efit": lambda: None, + # placeholder to stop missing_transforms error }) def read_data(self): @@ -954,7 +957,6 @@ def __call__( results.append(self.optimiser_context.format_results()) self.optimiser_context.optimiser.reset() - # unpack results and add time axis blobs = {} for key in results[0]["blobs"].keys(): @@ -982,15 +984,19 @@ def __call__( self.result = dict_of_dataarray_to_numpy(self.result) if mds_write: - # check_analysis_run(self.pulse, self.run) + tree_exists = does_tree_exist(pulse_to_write) + if tree_exists: + mode = "EDIT" + else: + mode = "NEW" + self.node_structure = create_nodes( pulse_to_write=pulse_to_write, run=run, diagnostic_quantities=self.blackbox_settings.opt_quantity, - mode="NEW", + mode=mode, ) write_nodes(pulse_to_write, self.node_structure, self.result) - return @@ -1042,7 +1048,7 @@ def __call__( data_settings = ReaderSettings(filters={}, revisions={}) # Add general methods for filtering data co-ords to ReadST40 data_context = MockData(pulse=pulse, diagnostics=diagnostics, - tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) + tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) # data_context = PhantomData(pulse=pulse, diagnostics=diagnostics, # tstart=tstart, tend=tend, dt=dt, reader_settings=data_settings, ) # data_context = ExpData(pulse=pulse, diagnostics=diagnostics, @@ -1077,7 +1083,8 @@ def __call__( optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=1000, sample_method="high_density", starting_samples=50, burn_frac=0.20, - stopping_criteria="mode", stopping_criteria_factor=0.002, stopping_criteria_debug=True, + stopping_criteria="mode", stopping_criteria_factor=0.002, + stopping_criteria_debug=True, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) From 1637d1ec101dbe8767765c1f0a1be12df79082d9 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 Jan 2024 09:49:49 +0000 Subject: [PATCH 273/288] check for tree and use NEW or EDIT mode --- indica/writers/bda_tree.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/indica/writers/bda_tree.py b/indica/writers/bda_tree.py index 243a8dd0..b75159a3 100644 --- a/indica/writers/bda_tree.py +++ b/indica/writers/bda_tree.py @@ -280,7 +280,7 @@ def write_nodes(pulse, node_info, data): ) -def check_analysis_run( +def check_to_overwrite_run( pulseNo, which_run, ): @@ -303,6 +303,17 @@ def check_analysis_run( overwrite_flag = query_yes_no(question) return overwrite_flag +def does_tree_exist(pulse, ): + IP_address_smaug = "smaug" + conn = Connection(IP_address_smaug) + + try: + conn.openTree("BDA", pulse) + conn.closeAllTrees() + return True + except: + return False + def query_yes_no( question, @@ -320,4 +331,13 @@ def query_yes_no( if __name__ == "__main__": - create_nodes(pulse_to_write=43000000, mode="EDIT", run="RUN01") + + pulse = 43000000 + run = "RUN01" + + tree_exists = does_tree_exist(pulse) + if tree_exists: + mode = "EDIT" + else: + mode = "NEW" + create_nodes(pulse_to_write=pulse, mode=mode, run=run) From b85bde83614f0875b47c66b260d55985d7eb49e3 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 Jan 2024 10:36:28 +0000 Subject: [PATCH 274/288] renaming Nimp -> niz1 --- indica/workflows/bayes_workflow.py | 56 +++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 68cacce7..319bee39 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -44,12 +44,12 @@ "Ne_prof.wcenter": 0.3, "Ne_prof.peaking": 1.2, - "Nimp_prof.y0": 1e17, - "Nimp_prof.y1": 1e15, - "Nimp_prof.yend": 1e15, - "Nimp_prof.wcenter": 0.3, - "Nimp_prof.wped": 3, - "Nimp_prof.peaking": 2, + "Niz1_prof.y0": 1e17, + "Niz1_prof.y1": 1e15, + "Niz1_prof.yend": 1e15, + "Niz1_prof.wcenter": 0.3, + "Niz1_prof.wped": 3, + "Niz1_prof.peaking": 2, "Te_prof.y0": 3000, "Te_prof.y1": 50, @@ -73,16 +73,16 @@ "Ne_prof.wped": loguniform(2, 20), "Ne_prof.wcenter": get_uniform(0.2, 0.4), "Ne_prof.peaking": get_uniform(1, 4), - "Nimp_prof.y0": loguniform(1e16, 1e18), - "Nimp_prof.y1": loguniform(1e14, 1e16), - "Ne_prof.y0/Nimp_prof.y0": lambda x1, x2: np.where( + "Niz1_prof.y0": loguniform(1e16, 1e18), + "Niz1_prof.y1": loguniform(1e14, 1e16), + "Ne_prof.y0/Niz1_prof.y0": lambda x1, x2: np.where( (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 ), - "Nimp_prof.y0/Nimp_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), - "Nimp_prof.wped": get_uniform(2, 6), - "Nimp_prof.wcenter": get_uniform(0.2, 0.4), - "Nimp_prof.peaking": get_uniform(1, 6), - "Nimp_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where( + "Niz1_prof.y0/Niz1_prof.y1": lambda x1, x2: np.where((x1 > x2), 1, 0), + "Niz1_prof.wped": get_uniform(2, 6), + "Niz1_prof.wcenter": get_uniform(0.2, 0.4), + "Niz1_prof.peaking": get_uniform(1, 6), + "Niz1_prof.peaking/Ne_prof.peaking": lambda x1, x2: np.where( (x1 > x2), 1, 0 ), # impurity always more peaked @@ -103,11 +103,11 @@ "Ne_prof.peaking", # "Ne_prof.wcenter", "Ne_prof.wped", - # "Nimp_prof.y1", - "Nimp_prof.y0", - # "Nimp_prof.wcenter", - # "Nimp_prof.wped", - "Nimp_prof.peaking", + # "Niz1_prof.y1", + "Niz1_prof.y0", + # "Niz1_prof.wcenter", + # "Niz1_prof.wped", + "Niz1_prof.peaking", "Te_prof.y0", "Te_prof.wped", "Te_prof.wcenter", @@ -157,11 +157,11 @@ # "Ne_prof.peaking", # "Ne_prof.wcenter", # "Ne_prof.wped", - # "Nimp_prof.y1", - # "Nimp_prof.y0", - # "Nimp_prof.wcenter", - # "Nimp_prof.wped", - # "Nimp_prof.peaking", + # "Niz1_prof.y1", + # "Niz1_prof.y0", + # "Niz1_prof.wcenter", + # "Niz1_prof.wped", + # "Niz1_prof.peaking", "Te_prof.y0", # "Te_prof.wped", # "Te_prof.wcenter", @@ -1035,10 +1035,10 @@ def __call__( "Ti_prof.peaking", "Ti_prof.wped", # "Ti_prof.wcenter", - # "Nimp_prof.y0", - # "Nimp_prof.peaking", - # "Nimp_prof.wcenter", - # "Nimp_prof.wped", + # "Niz1_prof.y0", + # "Niz1_prof.peaking", + # "Niz1_prof.wcenter", + # "Niz1_prof.wped", ] # BlackBoxSettings From 503fd5cb6d24004ea927f09f306d4be5859810c0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 22 Jan 2024 13:46:04 +0000 Subject: [PATCH 275/288] Plasma updating now works with Niz1/2_prof instead of Nimp_prof --- indica/models/plasma.py | 37 ++++++++++++++++++++---------- indica/workflows/bayes_workflow.py | 14 +++++------ 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/indica/models/plasma.py b/indica/models/plasma.py index 5f59a3e1..7b4a24b9 100644 --- a/indica/models/plasma.py +++ b/indica/models/plasma.py @@ -279,6 +279,8 @@ def initialize_variables(self): self.Ti_prof = Profiles(datatype=("temperature", "ion"), xspl=self.rho) self.Ne_prof = Profiles(datatype=("density", "electron"), xspl=self.rho) self.Nimp_prof = Profiles(datatype=("density", "impurity"), xspl=self.rho) + self.Niz1_prof = Profiles(datatype=("density", "impurity"), xspl=self.rho) + self.Niz2_prof = Profiles(datatype=("density", "impurity"), xspl=self.rho) self.Nh_prof = Profiles(datatype=("density", "thermal_neutral"), xspl=self.rho) self.Vrot_prof = Profiles(datatype=("rotation", "toroidal"), xspl=self.rho) @@ -519,12 +521,15 @@ def update_profiles( """ Update plasma profiles with profile parameters i.e. {"Ne_prof.y0":1e19} -> Ne_prof.y0 + TODO: refactor profiles into profiler structure and take care of initialisation of impurity profiles """ profile_prefixes: list = [ "Te_prof", "Ti_prof", "Ne_prof", - "Nimp_prof", + "Niz1_prof", + "Niz2_prof", + "Nh_prof", "Vrot_prof", ] for param, value in parameters.items(): @@ -542,17 +547,25 @@ def update_profiles( parameter_prefixes = [key.split(".")[0] for key in parameters.keys()] profile_names = set(parameter_prefixes) & set(profile_prefixes) - profile_mapping = { - "Te_prof": "electron_temperature", - "Ti_prof": "ion_temperature", - "Ne_prof": "electron_density", - "Vrot_prof": "toroidal_rotation", - "Nimp_prof": "impurity_density", - "Nh_prof": "neutral_density", - } - - for key in profile_names: - self.assign_profiles(profile_mapping[key], t=self.time_to_calculate) + if "Te_prof" in profile_names: + self.electron_temperature.loc[dict(t=self.time_to_calculate)] = self.Te_prof() + if "Ti_prof" in profile_names: + self.ion_temperature.loc[dict(t=self.time_to_calculate)] = self.Ti_prof() + if "Ne_prof" in profile_names: + self.electron_density.loc[dict(t=self.time_to_calculate)] = self.Ne_prof() + if "Nh_prof" in profile_names: + self.neutral_density.loc[dict(t=self.time_to_calculate)] = self.Nh_prof() + if "Vrot_prof" in profile_names: + self.electron_temperature.loc[dict(t=self.time_to_calculate)] = self.Ne_prof() + if "Niz1_prof" in profile_names: + self.impurity_density.loc[dict(t=self.time_to_calculate, element=self.impurities[0])] = self.Niz1_prof() + else: + self.impurity_density.loc[dict(t=self.time_to_calculate, element=self.impurities[0])] = self.Ne_prof() * self.impurity_concentration[0] + + if "Niz2_prof" in profile_names: + self.impurity_density.loc[dict(t=self.time_to_calculate, element=self.impurities[1])] = self.Niz2_prof() + else: + self.impurity_density.loc[dict(t=self.time_to_calculate, element=self.impurities[1])] = self.Ne_prof() * self.impurity_concentration[1] @property def time_to_calculate(self): diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 319bee39..0905d3c9 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -44,12 +44,12 @@ "Ne_prof.wcenter": 0.3, "Ne_prof.peaking": 1.2, - "Niz1_prof.y0": 1e17, - "Niz1_prof.y1": 1e15, - "Niz1_prof.yend": 1e15, - "Niz1_prof.wcenter": 0.3, - "Niz1_prof.wped": 3, - "Niz1_prof.peaking": 2, + # "Niz1_prof.y0": 1e17, + # "Niz1_prof.y1": 1e15, + # "Niz1_prof.yend": 1e15, + # "Niz1_prof.wcenter": 0.3, + # "Niz1_prof.wped": 3, + # "Niz1_prof.peaking": 2, "Te_prof.y0": 3000, "Te_prof.y1": 50, @@ -1081,7 +1081,7 @@ def __call__( data_context.process_data(model_context._build_bckc, ) - optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=1000, + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=100, sample_method="high_density", starting_samples=50, burn_frac=0.20, stopping_criteria="mode", stopping_criteria_factor=0.002, stopping_criteria_debug=True, From e0fed0e07a63cdff6800836ef1ca608be20b3029 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 24 Jan 2024 09:53:41 +0000 Subject: [PATCH 276/288] Changing default use case --- indica/workflows/bayes_workflow.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index 0905d3c9..cb5cad51 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -1007,7 +1007,7 @@ def __call__( dt = 0.01 diagnostics = [ - "xrcs", + # "xrcs", # "efit", # "smmh1", "cxff_pi", @@ -1016,7 +1016,7 @@ def __call__( ] # diagnostic_quantities opt_quant = [ - "xrcs.spectra", + # "xrcs.spectra", # "efit.wp", # "smmh1.ne", "cxff_pi.ti", @@ -1032,9 +1032,9 @@ def __call__( # "Te_prof.wped", # "Te_prof.wcenter", "Ti_prof.y0", - "Ti_prof.peaking", + # "Ti_prof.peaking", "Ti_prof.wped", - # "Ti_prof.wcenter", + "Ti_prof.wcenter", # "Niz1_prof.y0", # "Niz1_prof.peaking", # "Niz1_prof.wcenter", @@ -1081,9 +1081,9 @@ def __call__( data_context.process_data(model_context._build_bckc, ) - optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=10, iterations=100, + optimiser_settings = OptimiserEmceeSettings(param_names=bayes_settings.param_names, nwalkers=20, iterations=1000, sample_method="high_density", starting_samples=50, burn_frac=0.20, - stopping_criteria="mode", stopping_criteria_factor=0.002, + stopping_criteria="mode", stopping_criteria_factor=0.005, stopping_criteria_debug=True, priors=bayes_settings.priors) optimiser_context = EmceeOptimiser(optimiser_settings=optimiser_settings) From 3b124691659a8aeb9ae2e44b8be6563b6c8c07b0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 24 Jan 2024 11:02:41 +0000 Subject: [PATCH 277/288] Nimp_prof -> Niz1_prof --- indica/workflows/bda_run.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index 218a2097..527eca76 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -9,11 +9,11 @@ "Ne_prof.peaking", # "Ne_prof.wcenter", "Ne_prof.wped", - # "Nimp_prof.y1", - "Nimp_prof.y0", - # "Nimp_prof.wcenter", - # "Nimp_prof.wped", - "Nimp_prof.peaking", + # "Niz1_prof.y1", + "Niz1_prof.y0", + # "Niz1_prof.wcenter", + # "Niz1_prof.wped", + "Niz1_prof.peaking", "Te_prof.y0", "Te_prof.wped", "Te_prof.wcenter", @@ -30,11 +30,11 @@ # "Ne_prof.peaking", # "Ne_prof.wcenter", # "Ne_prof.wped", - # "Nimp_prof.y1", - "Nimp_prof.y0", - # "Nimp_prof.wcenter", - # "Nimp_prof.wped", - "Nimp_prof.peaking", + # "Niz1_prof.y1", + "Niz1_prof.y0", + # "Niz1_prof.wcenter", + # "Niz1_prof.wped", + "Niz1_prof.peaking", # "Te_prof.y0", # "Te_prof.wped", # "Te_prof.wcenter", From 5a62608a3f9e19b663c6e6af25c05b3bd3342ac5 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 24 Jan 2024 11:59:19 +0000 Subject: [PATCH 278/288] Option for fitting rho profiles as if symmetric in -rho --- indica/operators/gpr_fit.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/indica/operators/gpr_fit.py b/indica/operators/gpr_fit.py index c6e44061..707f1cb1 100644 --- a/indica/operators/gpr_fit.py +++ b/indica/operators/gpr_fit.py @@ -146,6 +146,7 @@ def gpr_fit_ts( xdim: str = "rho", virtual_obs=True, + virtual_symmetry=True, x_bounds=None, virtual_points=None, plot = False, @@ -159,7 +160,7 @@ def gpr_fit_ts( if virtual_points is None: if xdim is "rho": - virtual_points = [(-0.2, lambda y: np.nanmax(y)), (2.0, lambda y: 0), ] + virtual_points = [(1.5, lambda y: 0), ] elif xdim is "R": virtual_points = [(0, lambda y: 0), (0.9, lambda y: 0)] @@ -180,7 +181,13 @@ def gpr_fit_ts( num_vo = len(virtual_points) x = np.insert(x, [i for i in range(num_vo)], [virtual_point[0] for virtual_point in virtual_points]) y = np.insert(y, [i for i in range(num_vo)], [virtual_point[1](y) for virtual_point in virtual_points]) - y_err = np.insert(y_err, [i for i in range(num_vo)], [0.001 for i in range(num_vo)]) + y_err = np.insert(y_err, [i for i in range(num_vo)], [0.01 for i in range(num_vo)]) + + if virtual_symmetry: + x = np.concatenate((x, -x)) + y = np.concatenate((y, y)) + y_err = np.concatenate((y_err, y_err)) + _y_fit, _y_fit_err, _gpr = gpr_fit( x, @@ -218,18 +225,21 @@ def gpr_fit_ts( if __name__ == "__main__": - kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.5, 1.0), length_scale_bounds=(0.4, 0.7)) + kernels.WhiteKernel(noise_level_bounds=(0.01, 10)) - # kernel = kernels.RBF(length_scale_bounds=(0.1, 1.0)) + kernels.WhiteKernel(noise_level_bounds=(0.01, 10)) - - quant = "ne" pulse = 11089 - tstart = 0.05 - tend = 0.15 + tstart = 0.03 + tend = 0.12 dt = 0.01 - st40 = ReadST40(pulse, tstart, tend, dt) st40(instruments=["ts", "efit"]) - data = post_process_ts(st40.binned_data, st40.equilibrium, quant, pulse, split = "LFS",) - gpr_fit_ts(data=data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) + ne_data = post_process_ts(st40.binned_data, st40.equilibrium, "ne", pulse, split = "LFS",) + ne_kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.1, 1.0), length_scale_bounds=(0.4, 0.7)) + kernels.WhiteKernel(noise_level_bounds=(0.01, 10)) + + te_data = post_process_ts(st40.binned_data, st40.equilibrium, "te", pulse, split="LFS", ) + te_kernel = 1.0 * kernels.RationalQuadratic(alpha_bounds=(0.1, 1.0), + length_scale_bounds=(0.4, 0.7)) + kernels.WhiteKernel( + noise_level_bounds=(0.01, 10)) + + gpr_fit_ts(data=ne_data, xdim="rho", virtual_obs=True, virtual_symmetry=True, kernel=ne_kernel, save_fig=True) + gpr_fit_ts(data=te_data, xdim="rho", virtual_obs=True, virtual_symmetry=True, kernel=te_kernel, save_fig=True) From eb30028e3f9982b7e812acad2ada63d058fe5a7c Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Wed, 24 Jan 2024 12:00:10 +0000 Subject: [PATCH 279/288] Option for fitting rho profiles as if symmetric in -rho --- indica/workflows/bayes_workflow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index cb5cad51..e4451056 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -403,7 +403,8 @@ def fit_ts_profile(self, data_context, split="LFS", quant="ne"): processed_data = post_process_ts(deepcopy(data_context.binned_data), data_context.equilibrium, quant, data_context.pulse, split=split, ) - fit, _ = gpr_fit_ts(data=processed_data, xdim="rho", virtual_obs=True, kernel=kernel, save_fig=True) + fit, _ = gpr_fit_ts(data=processed_data, xdim="rho", virtual_obs=True, virtual_symmetry=True, + kernel=kernel, save_fig=True) fit = xr.where(fit < 0, 1e-10, fit) return fit From 3eefe45d51be04b09c3f485fa48ce1fcbf520c5a Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 26 Jan 2024 14:30:51 +0000 Subject: [PATCH 280/288] Alsu's 11XXX runs added --- indica/workflows/bda_run.py | 138 ++++++++++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 7 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index 527eca76..88f400ad 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -190,12 +190,75 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # dirname=f"{11336}_phantom", # )], - [(11089, - ["xrcs", "cxff_tws_c", "ts", "efit"], - PARAMS_SET_TS, + # [(11089, + # ["xrcs", "cxff_tws_c", "ts", "efit"], + # PARAMS_SET_TS, + # [ + # "xrcs.spectra", + # "cxff_tws_c.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=False, + # tstart=0.05, + # tend=0.10, + # dt=0.01, + # iterations=1000, + # nwalkers=20, + # stopping_criteria_factor=0.002, + # mds_write=True, + # plot=True, + # revisions={}, + # run="RUN01", + # # dirname="test", + # set_ts=True, + # )], + # RFX low ti/te pulse + + # [(11312, + # ["xrcs", "cxff_pi", "ts", "efit"], + # [ + # "Niz1_prof.y0", + # "Niz1_prof.peaking", + # "Ti_prof.y0", + # "Ti_prof.wcenter", + # "Ti_prof.peaking", + # ], + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=False, + # tstart=0.05, + # tend=0.10, + # dt=0.01, + # iterations=1000, + # nwalkers=20, + # stopping_criteria_factor=0.002, + # mds_write=True, + # plot=True, + # revisions={}, + # run="RUN01", + # # dirname="test", + # set_ts=True, + # )], + + [(11314, + ["xrcs", "cxff_pi", "ts", "efit"], + [ + "Niz1_prof.y0", + "Niz1_prof.peaking", + "Ti_prof.y0", + "Ti_prof.wcenter", + "Ti_prof.peaking", + ], [ "xrcs.spectra", - "cxff_tws_c.ti", + "cxff_pi.ti", "ts.ne", "ts.te", ]), @@ -204,9 +267,9 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, tstart=0.05, tend=0.10, dt=0.01, - iterations=1000, + iterations=2000, nwalkers=20, - stopping_criteria_factor=0.002, + stopping_criteria_factor=0.001, mds_write=True, plot=True, revisions={}, @@ -214,7 +277,68 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # dirname="test", set_ts=True, )], - # RFX low ti/te pulse + [(11317, + ["xrcs", "cxff_pi", "ts", "efit"], + [ + "Niz1_prof.y0", + "Niz1_prof.peaking", + "Ti_prof.y0", + "Ti_prof.wcenter", + "Ti_prof.peaking", + ], + [ + "xrcs.spectra", + "cxff_pi.ti", + "ts.ne", + "ts.te", + ]), + dict( + phantom=False, + tstart=0.05, + tend=0.10, + dt=0.01, + iterations=2000, + nwalkers=20, + stopping_criteria_factor=0.001, + mds_write=True, + plot=True, + revisions={}, + run="RUN01", + # dirname="test", + set_ts=True, + )], + + # [(11419, + # ["xrcs", "cxff_pi", "ts", "efit"], + # [ + # "Niz1_prof.y0", + # "Niz1_prof.peaking", + # "Ti_prof.y0", + # "Ti_prof.wcenter", + # "Ti_prof.peaking", + # ], + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=False, + # tstart=0.10, + # tend=0.16, + # dt=0.01, + # iterations=2000, + # nwalkers=20, + # stopping_criteria_factor=0.002, + # mds_write=True, + # plot=True, + # revisions={}, + # run="RUN01", + # # dirname="test", + # set_ts=True, + # )], + ] From 0db793baa0ee55f94665a876e8a10d5c2da091f2 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 26 Jan 2024 14:31:28 +0000 Subject: [PATCH 281/288] default values adjusted for Nimp --- indica/profiles_gauss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/profiles_gauss.py b/indica/profiles_gauss.py index 67b1cf7e..5837dff8 100644 --- a/indica/profiles_gauss.py +++ b/indica/profiles_gauss.py @@ -214,7 +214,7 @@ def get_defaults(datatype: tuple) -> dict: }, "impurity_density": { # (m**-3) "y0": 5.0e16, - "y1": 1.0e16, + "y1": 1.0e15, "yend": 1.0e15, "peaking": 2, "wcenter": 0.4, From d7dabeb0da726cda60f3de91a4bc96a0b4cb6ef0 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Fri, 26 Jan 2024 14:31:52 +0000 Subject: [PATCH 282/288] priors adjusted for Nimp --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index e4451056..c8e3713f 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -73,7 +73,7 @@ "Ne_prof.wped": loguniform(2, 20), "Ne_prof.wcenter": get_uniform(0.2, 0.4), "Ne_prof.peaking": get_uniform(1, 4), - "Niz1_prof.y0": loguniform(1e16, 1e18), + "Niz1_prof.y0": loguniform(2e15, 1e18), "Niz1_prof.y1": loguniform(1e14, 1e16), "Ne_prof.y0/Niz1_prof.y0": lambda x1, x2: np.where( (x1 > x2 * 100) & (x1 < x2 * 1e5), 1, 0 From 50cea66554835ea7c95fb8ff51f267cce6907eb9 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 5 Feb 2024 10:34:44 +0000 Subject: [PATCH 283/288] 11032 added --- indica/workflows/bda_run.py | 156 ++++++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 62 deletions(-) diff --git a/indica/workflows/bda_run.py b/indica/workflows/bda_run.py index 88f400ad..43cee46a 100644 --- a/indica/workflows/bda_run.py +++ b/indica/workflows/bda_run.py @@ -86,7 +86,7 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, tstart=0.02, tend=0.05, dt=0.01, revisions = {}, filters={}, iterations=500, nwalkers=50, stopping_criteria_factor=0.002, - mds_write=False, plot=False, run="RUN01", dirname=None, set_ts=False, **kwargs): + mds_write=False, plot=False, run="RUN01", dirname=None, set_ts=False, ts_split="LFS", **kwargs): if dirname is None: dirname = f"{pulse}" @@ -113,7 +113,7 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, plasma_context.save_phantom_profiles(phantoms=data_context.phantoms) if set_ts: - plasma_context.set_ts_profiles(data_context, split="LFS") + plasma_context.set_ts_profiles(data_context, split=ts_split) model_settings = ModelSettings(call_kwargs={"xrcs": {"pixel_offset": 0.0}}) @@ -247,66 +247,66 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # set_ts=True, # )], - [(11314, - ["xrcs", "cxff_pi", "ts", "efit"], - [ - "Niz1_prof.y0", - "Niz1_prof.peaking", - "Ti_prof.y0", - "Ti_prof.wcenter", - "Ti_prof.peaking", - ], - [ - "xrcs.spectra", - "cxff_pi.ti", - "ts.ne", - "ts.te", - ]), - dict( - phantom=False, - tstart=0.05, - tend=0.10, - dt=0.01, - iterations=2000, - nwalkers=20, - stopping_criteria_factor=0.001, - mds_write=True, - plot=True, - revisions={}, - run="RUN01", - # dirname="test", - set_ts=True, - )], - [(11317, - ["xrcs", "cxff_pi", "ts", "efit"], - [ - "Niz1_prof.y0", - "Niz1_prof.peaking", - "Ti_prof.y0", - "Ti_prof.wcenter", - "Ti_prof.peaking", - ], - [ - "xrcs.spectra", - "cxff_pi.ti", - "ts.ne", - "ts.te", - ]), - dict( - phantom=False, - tstart=0.05, - tend=0.10, - dt=0.01, - iterations=2000, - nwalkers=20, - stopping_criteria_factor=0.001, - mds_write=True, - plot=True, - revisions={}, - run="RUN01", - # dirname="test", - set_ts=True, - )], + # [(11314, + # ["xrcs", "cxff_pi", "ts", "efit"], + # [ + # "Niz1_prof.y0", + # "Niz1_prof.peaking", + # "Ti_prof.y0", + # "Ti_prof.wcenter", + # "Ti_prof.peaking", + # ], + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=False, + # tstart=0.05, + # tend=0.10, + # dt=0.01, + # iterations=2000, + # nwalkers=20, + # stopping_criteria_factor=0.001, + # mds_write=True, + # plot=True, + # revisions={}, + # run="RUN01", + # # dirname="test", + # set_ts=True, + # )], + # [(11317, + # ["xrcs", "cxff_pi", "ts", "efit"], + # [ + # "Niz1_prof.y0", + # "Niz1_prof.peaking", + # "Ti_prof.y0", + # "Ti_prof.wcenter", + # "Ti_prof.peaking", + # ], + # [ + # "xrcs.spectra", + # "cxff_pi.ti", + # "ts.ne", + # "ts.te", + # ]), + # dict( + # phantom=False, + # tstart=0.05, + # tend=0.10, + # dt=0.01, + # iterations=2000, + # nwalkers=20, + # stopping_criteria_factor=0.001, + # mds_write=True, + # plot=True, + # revisions={}, + # run="RUN01", + # # dirname="test", + # set_ts=True, + # )], # [(11419, # ["xrcs", "cxff_pi", "ts", "efit"], @@ -339,6 +339,38 @@ def bda_run(pulse, diagnostics, param_names, opt_quantity, phantom=False, # set_ts=True, # )], + [(11032, + ["xrcs", "cxff_pi", "ts", "efit"], + [ + "Niz1_prof.y0", + "Niz1_prof.peaking", + "Ti_prof.y0", + "Ti_prof.wcenter", + "Ti_prof.peaking", + ], + [ + "xrcs.spectra", + "cxff_pi.ti", + "ts.ne", + "ts.te", + ]), + dict( + phantom=False, + tstart=0.03, + tend=0.09, + dt=0.01, + iterations=2000, + nwalkers=20, + stopping_criteria_factor=0.001, + mds_write=True, + plot=True, + revisions={}, + run="RUN01", + # dirname="test", + set_ts=True, + ts_split="", + )], + ] From 6d3940881217bfe37017448b78cfb34188c5a5ef Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 5 Feb 2024 11:42:29 +0000 Subject: [PATCH 284/288] fixed missing self. --- indica/workflows/bayes_workflow.py | 6 +++--- ...est_bayes_workflow_example.py => test_bayes_workflow.py} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename tests/unit/workflows/{test_bayes_workflow_example.py => test_bayes_workflow.py} (100%) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index c8e3713f..c779c171 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -709,9 +709,9 @@ class MockData(PhantomData): def read_data(self): print("Reading mock equilibrium / transforms") self.equilibrium = fake_equilibrium( - tstart, - tend, - dt, + self.tstart, + self.tend, + self.dt, ) missing_transforms = list(set(diagnostics).difference(self.diagnostic_transforms.keys())) if missing_transforms: diff --git a/tests/unit/workflows/test_bayes_workflow_example.py b/tests/unit/workflows/test_bayes_workflow.py similarity index 100% rename from tests/unit/workflows/test_bayes_workflow_example.py rename to tests/unit/workflows/test_bayes_workflow.py From 174b7a4a7f498f05e42a8124a3c3eec10b9dac75 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 5 Feb 2024 11:47:00 +0000 Subject: [PATCH 285/288] fixed missing self. --- indica/workflows/bayes_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indica/workflows/bayes_workflow.py b/indica/workflows/bayes_workflow.py index c779c171..714f8a74 100644 --- a/indica/workflows/bayes_workflow.py +++ b/indica/workflows/bayes_workflow.py @@ -713,7 +713,7 @@ def read_data(self): self.tend, self.dt, ) - missing_transforms = list(set(diagnostics).difference(self.diagnostic_transforms.keys())) + missing_transforms = list(set(self.diagnostics).difference(self.diagnostic_transforms.keys())) if missing_transforms: raise ValueError(f"Missing transforms: {missing_transforms}") From 9c7e76d0e9fec88595cf1f4ab8d65a39cd6a3e8e Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 5 Feb 2024 15:29:59 +0000 Subject: [PATCH 286/288] initial commit --- .../workflows/test_bayes_workflow.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 tests/integration/workflows/test_bayes_workflow.py diff --git a/tests/integration/workflows/test_bayes_workflow.py b/tests/integration/workflows/test_bayes_workflow.py new file mode 100644 index 00000000..47e68ce3 --- /dev/null +++ b/tests/integration/workflows/test_bayes_workflow.py @@ -0,0 +1,78 @@ +import copy +from unittest.mock import MagicMock + +from indica.workflows.bayes_workflow import BayesWorkflow, PlasmaSettings, BayesBBSettings, ReaderSettings, MockData, \ + PlasmaContext, ModelSettings, ModelContext, OptimiserEmceeSettings, EmceeOptimiser +from indica.workflows.bayes_workflow import DEFAULT_PRIORS +from indica.workflows.bayes_workflow import DEFAULT_PROFILE_PARAMS + + +class TestBayesWorkflow: + def setup_class(self): + self.diagnostics = ["cxff_tws_c", "cxff_pi"] + self.opt_params = ["Ti_prof.y0", + "Ti_prof.peaking", + "Ti_prof.wped", + "Ti_prof.wcenter", ] + self.opt_quant = ["cxff_tws_c.ti", + "cxff_pi.ti"] + + self.pulse = None + self.tstart = 0.01 + self.tend = 0.02 + self.dt = 0.01 + + self.plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), + impurity_concentration=(0.001, 0.04), + n_rad=10) + + self.bayes_settings = BayesBBSettings(diagnostics=self.diagnostics, param_names=self.opt_params, + opt_quantity=self.opt_quant, priors=DEFAULT_PRIORS, ) + + self.data_settings = ReaderSettings(filters={}, + revisions={}) + + self.model_settings = ModelSettings(call_kwargs={"xrcs": {"pixel_offset": 0.0}}) + + self.optimiser_settings = OptimiserEmceeSettings(param_names=self.bayes_settings.param_names, nwalkers=10, + iterations=10, + sample_method="random", starting_samples=10, + burn_frac=0.00, + stopping_criteria="mode", stopping_criteria_factor=0.005, + stopping_criteria_debug=True, + priors=self.bayes_settings.priors) + + def test_workflow_runs(self): + + data_context = MockData(pulse=self.pulse, diagnostics=self.diagnostics, + tstart=self.tstart, tend=self.tend, + dt=self.dt, reader_settings=self.data_settings, ) + data_context.read_data() + + plasma_context = PlasmaContext(plasma_settings=self.plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) + plasma_context.init_plasma(data_context.equilibrium, self.tstart, self.tend, self.dt) + plasma_context.save_phantom_profiles(phantoms=data_context.phantoms) + + model_context = ModelContext(diagnostics=self.diagnostics, + plasma_context=plasma_context, + equilibrium=data_context.equilibrium, + transforms=data_context.transforms, + model_settings=self.model_settings, ) + model_context.update_model_kwargs(data_context.binned_data) + model_context.init_models() + data_context.process_data(model_context._build_bckc, ) + + optimiser_context = EmceeOptimiser(optimiser_settings=self.optimiser_settings) + + workflow = BayesWorkflow(tstart=self.tstart, tend=self.tend, dt=self.dt, + blackbox_settings=self.bayes_settings, data_context=data_context, + optimiser_context=optimiser_context, + plasma_context=plasma_context, model_context=model_context) + workflow(pulse_to_write=43000001, run="TEST", mds_write=False, plot=False, filepath=f"./results/test/") + + +if __name__ == "__main__": + + test = TestBayesWorkflow() + test.setup_class() + test.test_workflow_runs() \ No newline at end of file From 718e0917a0749ed8b88b96ce35d63cedae7b20e9 Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 5 Feb 2024 15:30:49 +0000 Subject: [PATCH 287/288] mocking opt_data keys --- tests/unit/workflows/test_bayes_workflow.py | 234 ++++++++------------ 1 file changed, 91 insertions(+), 143 deletions(-) diff --git a/tests/unit/workflows/test_bayes_workflow.py b/tests/unit/workflows/test_bayes_workflow.py index 383f19c8..64a63fba 100644 --- a/tests/unit/workflows/test_bayes_workflow.py +++ b/tests/unit/workflows/test_bayes_workflow.py @@ -1,160 +1,108 @@ import copy - import pytest +from unittest.mock import MagicMock -from indica.workflows.bayes_workflow_example import BayesWorkflowExample -from indica.workflows.bayes_workflow_example import DEFAULT_PRIORS -from indica.workflows.bayes_workflow_example import DEFAULT_PROFILE_PARAMS -from indica.workflows.bayes_workflow_example import OPTIMISED_PARAMS -from indica.workflows.bayes_workflow_example import OPTIMISED_QUANTITY - -""" -TODO: -Mock reader for testing experimental data reading -""" +from indica.workflows.bayes_workflow import BayesWorkflow, PlasmaSettings, BayesBBSettings, ReaderSettings, MockData, \ + PlasmaContext, ModelSettings, ModelContext, OptimiserEmceeSettings, EmceeOptimiser +from indica.workflows.bayes_workflow import DEFAULT_PRIORS +from indica.workflows.bayes_workflow import DEFAULT_PROFILE_PARAMS -class TestBayesWorkflowExample: +class TestBayesWorkflow: def setup_class(self): - self.init_settings = dict( - pulse=None, - phantoms=True, - diagnostics=["xrcs", "efit", "smmh1", "cxff_pi"], - opt_quantity=OPTIMISED_QUANTITY, - param_names=OPTIMISED_PARAMS, - profile_params=DEFAULT_PROFILE_PARAMS, - priors=DEFAULT_PRIORS, - tstart=0.02, - tend=0.10, - dt=0.005, - ) - self.plasma_settings = dict( - tsample=0.060, - ) - - self.optimiser_settings = dict( - model_kwargs={ - "xrcs_moment_analysis": False, - }, - nwalkers=20, - sample_high_density=False, - ) - - self.call_settings = dict( - filepath=None, - pulse_to_write=23000101, - run="RUN01", - mds_write=False, - plot=False, - iterations=1, - burn_frac=0.10, - ) - self.sampler_settings = dict( - iterations=1, - burn_frac=0.10, - ) - - self.workflow_untouched = BayesWorkflowExample(**self.init_settings) - self.workflow = None - - def setup_method(self): - self.workflow = copy.deepcopy(self.workflow_untouched) - - def teardown_method(self): - self.workflow = None - - def test_workflow_initializes(self): - attributes_to_check = ["data", "models", "equilibrium"] - for attribute in attributes_to_check: - if not hasattr(self.workflow, attribute): - raise ValueError(f"missing {attribute} in workflow object") + self.diagnostics = ["cxff_tws_c", "cxff_pi"] + self.opt_params = ["Ti_prof.y0", + "Ti_prof.peaking", + "Ti_prof.wped", + "Ti_prof.wcenter", ] + self.opt_quant = ["cxff_tws_c.ti", + "cxff_pi.ti"] + + self.pulse = None + self.tstart = 0.01 + self.tend = 0.10 + self.dt = 0.01 + self.phantoms = True + + self.plasma_settings = PlasmaSettings(main_ion="h", impurities=("ar", "c"), + impurity_concentration=(0.001, 0.04), + n_rad=10) + + self.bayes_settings = BayesBBSettings(diagnostics=self.diagnostics, param_names=self.opt_params, + opt_quantity=self.opt_quant, priors=DEFAULT_PRIORS, ) + + self.data_settings = ReaderSettings(filters={}, + revisions={}) + + self.model_settings = ModelSettings(call_kwargs={"xrcs": {"pixel_offset": 0.0}}) + + self.optimiser_settings = OptimiserEmceeSettings(param_names=self.bayes_settings.param_names, nwalkers=10, + iterations=10, + sample_method="random", starting_samples=10, + burn_frac=0.05, + stopping_criteria="mode", stopping_criteria_factor=0.005, + stopping_criteria_debug=True, + priors=self.bayes_settings.priors) + + self.data_context = MagicMock(pulse=self.pulse, diagnostics=self.diagnostics, + tstart=self.tstart, tend=self.tend, + dt=self.dt, reader_settings=self.data_settings, ) + self.data_context.opt_data.keys = MagicMock(return_value=self.opt_quant) + + self.plasma_context = MagicMock(plasma_settings=self.plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) + + self.model_context = MagicMock(diagnostics=self.diagnostics, + plasma_context=self.plasma_context, + equilibrium=self.data_context.equilibrium, + transforms=self.data_context.transforms, + model_settings=self.model_settings, ) + + self.optimiser_context = MagicMock(optimiser_settings=self.optimiser_settings) + + self.workflow = MagicMock(tstart=self.tstart, tend=self.tend, dt=self.dt, + blackbox_settings=self.bayes_settings, data_context=self.data_context, + optimiser_context=self.optimiser_context, + plasma_context=self.plasma_context, model_context=self.model_context) + + def test_data_context_initialises(self): + data_context = MockData(pulse=self.pulse, diagnostics=self.diagnostics, + tstart=self.tstart, tend=self.tend, + dt=self.dt, reader_settings=self.data_settings, ) + data_context.read_data() + data_context.process_data(self.model_context._build_bckc, ) + assert True + + def test_plasma_context_initialises(self): + plasma_context = PlasmaContext(plasma_settings=self.plasma_settings, profile_params=DEFAULT_PROFILE_PARAMS) + plasma_context.init_plasma(equilibrium=self.data_context.equilibrium, tstart=self.tstart, tend=self.tend, + dt=self.dt, ) + plasma_context.save_phantom_profiles(phantoms=self.data_context.phantoms) assert True - def test_init_phantoms_false_with_example_plasma(self): - with pytest.raises(ValueError): - BayesWorkflowExample(**dict(self.init_settings, **{"phantoms": False})) - - def test_init_not_including_all_required_inputs(self): - with pytest.raises(ValueError): - BayesWorkflowExample(**dict(self.init_settings, **{"param_names": None})) - - # def test_reader_has_read_all_diagnostic_data(self): - # assert all(diag_name in self.workflow.reader.keys() - # for diag_name in self.workflow.diagnostics) - - def test_plasma_has_equilibrium(self): - self.workflow.setup_plasma(**self.plasma_settings) - assert hasattr(self.workflow.plasma, "equilibrium") - - def test_phantom_profiles_are_not_mutatable(self): - self.workflow.setup_plasma(**self.plasma_settings) - phantoms = copy.deepcopy(self.workflow.phantom_profiles) - self.workflow.plasma.electron_temperature += 1 - assert phantoms is not self.workflow.phantom_profiles - - def test_setup_models_with_wrong_diagnostic_names(self): - with pytest.raises(ValueError): - self.workflow.setup_models(["foo", "bar", "xrcs"]) - - def test_opt_data_without_plasma(self): - with pytest.raises(ValueError): - self.workflow.setup_opt_data(phantoms=True) - - def test_phantom_data_exists(self): - self.workflow.setup_plasma(**self.plasma_settings) - self.workflow.setup_opt_data(phantoms=True) - assert self.workflow.opt_data - - # def test_experimental_data_exists(self): - # self.workflow._exp_data() - # assert self.workflow.opt_data - - def test_phantom_data_has_time_dim(self): - self.workflow.setup_plasma(**self.plasma_settings) - self.workflow.setup_opt_data(phantoms=True) - for key, value in self.workflow.opt_data.items(): - assert "t" in value.dims - - # def test_experimental_data_has_time_dim(self): - # self.workflow._exp_data() - # for key, value in self.workflow.opt_data.items(): - # assert "t" in value.dims - - def test_phantom_data_runs_with_noise_added(self): - self.workflow.setup_plasma(**self.plasma_settings) - self.workflow.setup_opt_data(phantoms=True, noise=True) - assert self.workflow.opt_data - - def test_sampling_from_priors(self): - self.workflow.setup_plasma(**self.plasma_settings) - self.workflow.setup_opt_data( - phantoms=True, - ) - self.workflow.setup_optimiser( - **dict(self.optimiser_settings, **{"sample_high_density": False}) - ) + def test_model_context_initialises(self): + model_context = ModelContext(diagnostics=self.diagnostics, + plasma_context=self.plasma_context, + equilibrium=self.data_context.equilibrium, + transforms=self.data_context.transforms, + model_settings=self.model_settings, + ) + model_context.update_model_kwargs(self.data_context.binned_data) + model_context.init_models() assert True - def test_sampling_from_high_density(self): - self.workflow.setup_plasma(**self.plasma_settings) - self.workflow.setup_opt_data( - phantoms=True, - ) - self.workflow.setup_optimiser( - **dict(self.optimiser_settings, **{"sample_high_density": True}) - ) + def test_optimiser_context_initialises(self): + optimiser_context = EmceeOptimiser(optimiser_settings=self.optimiser_settings) assert True - def test_worklow_has_results_after_run(self): - self.workflow.setup_plasma(**self.plasma_settings) - self.workflow.setup_opt_data(phantoms=True) - self.workflow.setup_optimiser(**self.optimiser_settings) - self.workflow.run_sampler(**self.sampler_settings) - if not hasattr(self.workflow, "result"): - raise ValueError("missing result in workflow object") + def test_workflow_initialises(self): + workflow = BayesWorkflow(tstart=self.tstart, tend=self.tend, dt=self.dt, + blackbox_settings=self.bayes_settings, data_context=self.data_context, + optimiser_context=self.optimiser_context, + plasma_context=self.plasma_context, model_context=self.model_context) assert True if __name__ == "__main__": - test = TestBayesWorkflowExample() + test = TestBayesWorkflow() test.setup_class() + test.test_workflow_initialises() From ac148e9c841d0e6682d1b66650352bd005b6784d Mon Sep 17 00:00:00 2001 From: michael gemmell Date: Mon, 5 Feb 2024 15:32:51 +0000 Subject: [PATCH 288/288] speeding up run by limiting iterations --- .../integration/workflows/test_bayes_workflow.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/integration/workflows/test_bayes_workflow.py b/tests/integration/workflows/test_bayes_workflow.py index 47e68ce3..214a5ec6 100644 --- a/tests/integration/workflows/test_bayes_workflow.py +++ b/tests/integration/workflows/test_bayes_workflow.py @@ -11,9 +11,10 @@ class TestBayesWorkflow: def setup_class(self): self.diagnostics = ["cxff_tws_c", "cxff_pi"] self.opt_params = ["Ti_prof.y0", - "Ti_prof.peaking", - "Ti_prof.wped", - "Ti_prof.wcenter", ] + # "Ti_prof.peaking", + # "Ti_prof.wped", + # "Ti_prof.wcenter", + ] self.opt_quant = ["cxff_tws_c.ti", "cxff_pi.ti"] @@ -34,10 +35,10 @@ def setup_class(self): self.model_settings = ModelSettings(call_kwargs={"xrcs": {"pixel_offset": 0.0}}) - self.optimiser_settings = OptimiserEmceeSettings(param_names=self.bayes_settings.param_names, nwalkers=10, - iterations=10, - sample_method="random", starting_samples=10, - burn_frac=0.00, + self.optimiser_settings = OptimiserEmceeSettings(param_names=self.bayes_settings.param_names, nwalkers=5, + iterations=2, + sample_method="random", starting_samples=2, + burn_frac=0, stopping_criteria="mode", stopping_criteria_factor=0.005, stopping_criteria_debug=True, priors=self.bayes_settings.priors)