From 9550ffe3043a68930296ca7a33895b1eb466b781 Mon Sep 17 00:00:00 2001 From: Adam Theisen Date: Fri, 13 Oct 2023 11:31:43 -0500 Subject: [PATCH] =?UTF-8?q?ADD:=20Adding=20new=20example=20for=20working?= =?UTF-8?q?=20with=20ARM=20QC=20and=20Xarray=20Transforma=E2=80=A6=20(#734?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ADD: Adding new example for working with ARM QC and Xarray Transformations. Updating one example to follow ds convention, and fixing warn message for armfiles * ENH: PEP8 update * DOC: test updates --- act/io/armfiles.py | 2 +- examples/qc/plot_arm_qc.py | 30 +++++++-------- examples/workflows/plot_qc_transforms.py | 48 ++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 16 deletions(-) create mode 100644 examples/workflows/plot_qc_transforms.py diff --git a/act/io/armfiles.py b/act/io/armfiles.py index c2f362b1ef..9e591eae09 100644 --- a/act/io/armfiles.py +++ b/act/io/armfiles.py @@ -105,7 +105,7 @@ def read_netcdf( """ - message = 'act.io.armfiles.read_netcdf will be replaced in version 2.0.0 by act.io.arm.read_netcdf()' + message = 'act.io.armfiles.read_netcdf will be replaced in version 2.0.0 by act.io.arm.read_arm_netcdf()' warnings.warn(message, DeprecationWarning, 2) diff --git a/examples/qc/plot_arm_qc.py b/examples/qc/plot_arm_qc.py index 63e21b454b..c775fec1ec 100644 --- a/examples/qc/plot_arm_qc.py +++ b/examples/qc/plot_arm_qc.py @@ -43,11 +43,11 @@ # the cleanup_qc keyword. This will convert the quality control variable from the ARM stanard # to Climate and Forecast standard used internally for all the quality control calls. keep_vars = [variable, qc_variable, 'lat', 'lon'] -obj = act.io.armfiles.read_netcdf(results, keep_variables=keep_vars, cleanup_qc=True) -print(obj) +ds = act.io.armfiles.read_netcdf(results, keep_variables=keep_vars, cleanup_qc=True) +print(ds) # Create a plotting display object with 2 plots -display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,)) +display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,)) # Plot up the diffuse variable in the first plot display.plot(variable, subplot_index=(0,), day_night_background=True) @@ -62,10 +62,10 @@ # By default the ancillary quality control variable is removed after appying the test # results, but we are going to use the del_qc_var to keep in Dataset so it # can be used with additional tests later. -obj.qcfilter.datafilter(variable, rm_tests=[2, 3], del_qc_var=False) +ds.qcfilter.datafilter(variable, rm_tests=[2, 3], del_qc_var=False) # Create a plotting display object with 2 plots -display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,)) +display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,)) # Plot up the diffuse variable in the first plot display.plot(variable, subplot_index=(0,), day_night_background=True) @@ -84,10 +84,10 @@ # Query the ARM DQR Webservice and update the ancillary quality control variable to # contain a new test using information from the DQR. -obj = act.qc.arm.add_dqr_to_qc(obj, variable=variable) +ds = act.qc.arm.add_dqr_to_qc(ds, variable=variable) # Create a plotting display object with 2 plots -display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,)) +display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,)) # Plot up the diffuse variable in the first plot display.plot(variable, subplot_index=(0,), day_night_background=True) @@ -102,13 +102,13 @@ # going to filter the data based on this new test and plot up the results. # Add a new maximum tests -obj.qcfilter.add_greater_test(variable, 0.4, test_meaning='New maximum tests limit') +ds.qcfilter.add_greater_test(variable, 0.4, test_meaning='New maximum tests limit') # Filter that test out -obj.qcfilter.datafilter(variable, rm_tests=5, del_qc_var=False) +ds.qcfilter.datafilter(variable, rm_tests=5, del_qc_var=False) # Create a plotting display object with 2 plots -display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,)) +display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,)) # Plot up the diffuse variable in the first plot display.plot(variable, subplot_index=(0,), day_night_background=True) @@ -123,10 +123,10 @@ # it is applied in a moving window style approach. # Apply test -obj = act.qc.fft_shading_test(obj, variable=variable) +ds = act.qc.fft_shading_test(ds, variable=variable) # Create a plotting display object with 2 plots -display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,)) +display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,)) # Plot up the diffuse variable in the first plot display.plot(variable, subplot_index=(0,), day_night_background=True) @@ -146,16 +146,16 @@ # incorrect or suspect values that can be read and applied to the Dataset. from act.qc.add_supplemental_qc import apply_supplemental_qc -apply_supplemental_qc(obj, 'sgpmfrsr7nchE11.b1.yaml') +apply_supplemental_qc(ds, 'sgpmfrsr7nchE11.b1.yaml') # We can apply or reapply the data filter on the variable in the Dataset to change # the data values failing tests to NaN by passing a list of test numbers we want # to use. In this case we are not going to apply the DQR test (number 4) so we leave # that number out of the list. -obj.qcfilter.datafilter(variable, rm_tests=[2, 3, 5, 6, 7, 8], del_qc_var=False) +ds.qcfilter.datafilter(variable, rm_tests=[2, 3, 5, 6, 7, 8], del_qc_var=False) # Create a plotting display object with 2 plots -display = act.plotting.TimeSeriesDisplay(obj, figsize=(15, 10), subplot_shape=(2,)) +display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(2,)) # Plot up the diffuse variable in the first plot display.plot(variable, subplot_index=(0,), day_night_background=True) diff --git a/examples/workflows/plot_qc_transforms.py b/examples/workflows/plot_qc_transforms.py new file mode 100644 index 0000000000..903c53da4f --- /dev/null +++ b/examples/workflows/plot_qc_transforms.py @@ -0,0 +1,48 @@ +""" +Transformations and QC +---------------------- + +Built-in transformations using xarray are not +quality-control aware. This example shows how +a user should apply QC prior to performing transformations. + +""" + +import act +import xarray as xr +import matplotlib.pyplot as plt + +# Read in some sample MFRSR data and clean up the QC +ds = act.io.armfiles.read_netcdf(act.tests.sample_files.EXAMPLE_MFRSR, cleanup_qc=True) + +# Let's resample the data to 5 minutes and take the mean +ds_5min = ds.resample(time='5min').mean() + +variable = 'diffuse_hemisp_narrowband_filter4' + +# Let's look at a before and after of one of the qc variables +print('With no QC applied before transformation') +print('Before (10 1-minute samples): ', ds['qc_' + variable].values[0:10]) +print('After: (2 5-minute averages)', ds_5min['qc_' + variable].values[0:2]) + +# That new QC variable does not make sense at all and should be an int +# What needs to happen is that we apply QC as the user see's fit to all +# variables before the transformations take place. +print('\nAverage of ', variable, ' before and after applying QC') +print('Note the change in the second value') +print('Before (2 5 - minute averages): ', ds[variable].values[0:2]) + +ds.qcfilter.datafilter(rm_assessments=['Bad', 'Indeterminate']) +ds_5minb = ds.resample(time='5min').mean() + +# Print out the corresponding variable values +print('After: (2 5 - minute averages)', ds_5minb[variable].values[0:2]) + +## Plot up the variable and qc block plot +display = act.plotting.TimeSeriesDisplay({'Original': ds, 'Average': ds_5min, 'Average_QCd': ds_5minb}, + figsize=(15, 10), subplot_shape=(2,)) +display.plot(variable, dsname='Original', subplot_index=(0,), day_night_background=True) +display.plot(variable, dsname='Average', subplot_index=(1,), day_night_background=True, label='No QC') +display.plot(variable, dsname='Average_QCd', subplot_index=(1,), day_night_background=True, label='QC') +plt.legend() +plt.show()