Add more tests (#3)

* add flash clustering tests

* add pyproj to test dependencies

* update deps again

* add filter tests

* hey look, a bug fix, just from improving code coverage!

* same fix as in wx4stg/lmatools@468469f

* no prune filter test

* use correct python version

* test gridding

* add geosys and eqc tests

* radar and satellite

* test plot feature

* reorganize, maybe this makes more sense?

* init matplotlib tests

* fix plotting if no metpy

* run image tests

* pytest-mpl dep

* that doesn't work the way I thought it did...

* add plot_feature mpl tests

* artifact matplotlib, maybe?

* -_-

* >:/

* 3rd = charm

* if 3rd is charm, 4th is perfect.. right?

* add county lines back to the images

* inset_view test

* this feels like a duplicate of the base plot..

* test the static part of the interactivity, anyway.

* more deps!

.github/workflows/python-package.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v3
@@ -29,15 +29,21 @@ jobs:
         "${SHELL}" <(curl -L micro.mamba.pm/install.sh)
         eval "$(micromamba shell hook --shell bash)"
         micromamba activate base
-        micromamba install pytest-cov xarray netcdf4 -c conda-forge
+        micromamba install pytest-cov pytest-mpl xarray netcdf4 pandas numpy scikit-learn scipy pyproj cartopy metpy ipywidgets python=${{ matrix.python-version }} -c conda-forge
         python -m pip install .
     - name: Test with pytest
       run: |
         eval "$(micromamba shell hook --shell bash)"
         micromamba activate base
-        coverage run --source=pyxlma -m pytest tests/
+        coverage run --source=pyxlma -m pytest --mpl --mpl-baseline-path=tests/truth/images/ --mpl-generate-summary=html,json --mpl-results-path=tests/mpl-results/ tests/
         coverage xml
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v3
       env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+    - name: Upload matplotlib test results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: matplotlib-results
+        path: tests/mpl-results/

README.md

@@ -16,6 +16,38 @@ pip install -e .
 
 Then, copy the `XLMA_plots.ipynb` notebook to wherever you'd like and start changing files, dates and times to show data from your case of interest. There also a notebook showing how to do flash sorting and save a new NetCDF file with those data.
 
+# Dependencies
+Required:
+
+- xarray (I/O requires the netcdf4 backend)
+- pandas
+- numpy
+
+Flash clustering:
+
+- scikit-learn
+- scipy
+- pyproj
+
+Plotting:
+
+- matplotlib
+- cartopy
+- metpy (optionally, for US county lines)
+
+Interactive:
+
+- jupyterlab (or, notebook)
+- ipywidgets
+- ipympl
+
+Building:
+
+- setuptools
+- pytest-cov
+- pytest-mpl
+- ...and all of the above
+
 # Technical architecture
 
 We envision a two-part package that keeps a clean separation between the core data model, analysis, and display. XLMA utilized a large, global `state` structure that stored all data, as well as the current data selection corresponding to the view in the GUI. Analysis then operated on whatever data was in the current selection.

pyxlma/coords.py

@@ -67,6 +67,8 @@ def __init__(self, ellipse='WGS84', datum='WGS84',
         if (r_equator is not None) | (r_pole is not None):
             if r_pole is None:
                 r_pole=r_equator
+            if r_equator is None:
+                r_equator=r_pole
             self.ERSlla = proj4.Proj(proj='latlong', a=r_equator, b=r_pole)
             self.ERSxyz = proj4.Proj(proj='geocent', a=r_equator, b=r_pole)
         else:

pyxlma/lmalib/flash/properties.py

@@ -19,7 +19,7 @@ def hull_volume(xyz):
     assert xyz.shape[1] == 3
 
     tri = Delaunay(xyz[:,0:3])
-    vertices = tri.points[tri.vertices]
+    vertices = tri.points[tri.simplices]
 
     # This is the volume formula in
     # https://github.com/scipy/scipy/blob/master/scipy/spatial/tests/test_qhull.py#L106
@@ -287,7 +287,7 @@ def filter_flashes(ds, **kwargs):
     good = np.ones(ds.flash_id.shape, dtype=bool)
     # print("Starting flash count: ", good.sum())
     if 'flash_event_count' in ds.variables:
-        if np.all(ds.flash_event_count == np.iinfo(np.uint64).max):
+        if np.all(ds.flash_event_count == np.iinfo(np.uint32).max):
             raise ValueError('Before filtering a dataset by flash properties, call flash_stats on the dataset to compute flash properties.')
     for v, (vmin, vmax) in kwargs.items():
         if vmin is not None:

pyxlma/plot/xlma.py

@@ -2,6 +2,7 @@
 import matplotlib.pyplot as plt
 import matplotlib.dates as md
 import datetime as dt
+import pandas as pd
 from matplotlib.ticker import Formatter, FormatStrFormatter, MaxNLocator
 
 import cartopy
@@ -145,7 +146,7 @@ def setup_figure(self, **kwargs):
             from pandas.plotting import register_matplotlib_converters
             register_matplotlib_converters()
             try:
-                self.datetime = self.data.event_time.to_dataframe().event_time
+                self.datetime = pd.Series(self.data.event_time.data.flatten()).dt.to_pydatetime()
             except:
                 self.data_exists=False
                 self.datetime = [self.stime,self.stime+dt.timedelta(minutes=10)]

pyxlma/plot/xlma_base_plot.py

@@ -181,7 +181,8 @@ def plot(self, **kwargs):
 
         # Plan view
         if self.bkgmap==True:
-            self.ax_plan.add_feature(COUNTIES, facecolor='none', edgecolor='gray')
+            if COUNTIES != None:
+                self.ax_plan.add_feature(COUNTIES, facecolor='none', edgecolor='gray')
             self.ax_plan.add_feature(cfeature.BORDERS)
             self.ax_plan.add_feature(cfeature.STATES.with_scale('10m'))
         self.ax_plan.set_xlabel('Longitude (degrees)')

pyxlma/plot/xlma_plot_feature.py

@@ -67,19 +67,19 @@ def plot_points(bk_plot, lon_data, lat_data, alt_data, time_data,
     # before **kwargs was added to the function call, the following arguments
     # were specified as keywords separately. This allows backwards compatibility:
     if plot_cmap is None:
-        plot_cmap = kwargs.pop('cmap', kwargs.pop('plot_cmap', None))
+        plot_cmap = kwargs.pop('cmap', plot_cmap)
     if plot_s is None:
-        plot_s = kwargs.pop('s', kwargs.pop('plot_s', None))
+        plot_s = kwargs.pop('s', plot_s)
     if plot_vmin is None:
-        plot_vmin = kwargs.pop('vmin', kwargs.pop('plot_vmin', None))
+        plot_vmin = kwargs.pop('vmin', plot_vmin)
     if plot_vmax is None:
-        plot_vmax = kwargs.pop('vmax', kwargs.pop('plot_vmax', None))
+        plot_vmax = kwargs.pop('vmax', plot_vmax)
     if plot_c is None:
-        plot_c = kwargs.pop('c', kwargs.pop('plot_c', None))
+        plot_c = kwargs.pop('c', plot_c)
     if edge_color == 'face':
-        edge_color = kwargs.pop('edgecolors', kwargs.pop('edge_color', 'face'))
+        edge_color = kwargs.pop('edgecolors', edge_color)
     if edge_width == 0:
-        edge_width = kwargs.pop('linewidths', kwargs.pop('edge_width', 0))
+        edge_width = kwargs.pop('linewidths', edge_width)
 
     art_plan = bk_plot.ax_plan.scatter(lon_data, lat_data,
                             c=plot_c,vmin=plot_vmin, vmax=plot_vmax, cmap=plot_cmap,
@@ -119,7 +119,7 @@ def plot_3d_grid(bk_plot, xedges, yedges, zedges, tedges,
 
     """
 
-    plot_cmap = kwargs.pop('cmap', kwargs.pop('plot_cmap', None))
+    plot_cmap = kwargs.pop('cmap', plot_cmap)
     plot_vmin = kwargs.pop('vmin', 0)
 
     alt_lon[alt_lon==0]=np.nan

tests/test_base_plot.py

@@ -0,0 +1,124 @@
+import pytest
+import xarray as xr
+from pyxlma.plot.xlma_base_plot import *
+from pyxlma.plot.xlma_plot_feature import *
+from pyxlma.lmalib.grid import *
+import datetime as dt
+import pandas as pd
+import matplotlib.dates as md
+
+@pytest.mark.mpl_image_compare
+def test_blank_plot():
+    start_time = dt.datetime(2023, 12, 24, 0, 57, 0, 0)
+    end_time = start_time + dt.timedelta(seconds=60)
+    bk_plot = BlankPlot(start_time, bkgmap=True, xlim=[-103.5, -99.5], ylim=[31.5, 35.5], zlim=[0, 20], tlim=[start_time, end_time], title='XLMA Test Plot')
+    return bk_plot.fig
+
+@pytest.mark.mpl_image_compare
+def test_blank_plot_labeled():
+    start_time = dt.datetime(2023, 12, 24, 0, 57, 0, 0)
+    end_time = start_time + dt.timedelta(seconds=60)
+    bk_plot = BlankPlot(start_time, bkgmap=True, xlim=[-103.5, -99.5], ylim=[31.5, 35.5], zlim=[0, 20], tlim=[start_time, end_time], title='XLMA Test Plot')
+    subplot_labels(bk_plot)
+    return bk_plot.fig
+
+@pytest.mark.mpl_image_compare
+def test_plot_feature_plot_points_positional():
+    start_time = dt.datetime(2023, 12, 24, 0, 57, 0, 0)
+    end_time = start_time + dt.timedelta(seconds=60)
+    bk_plot = BlankPlot(start_time, bkgmap=True, xlim=[-103.5, -99.5], ylim=[31.5, 35.5], zlim=[0, 20], tlim=[start_time, end_time], title='XLMA Test Plot')
+    dataset = xr.open_dataset('tests/truth/lma_netcdf/lma.nc')
+    times = pd.Series(dataset.event_time.data.flatten())
+    vmin, vmax, colors = color_by_time(pd.to_datetime(times), (start_time, end_time))
+    plot_points(bk_plot, dataset.event_longitude.data, dataset.event_latitude.data, dataset.event_altitude.data/1000,
+                dataset.event_time.data, 'rainbow', 5, vmin, vmax, colors, 'k', 0.1, True)
+    return bk_plot.fig
+
+@pytest.mark.mpl_image_compare
+def test_plot_feature_plot_points_old_kw():
+    start_time = dt.datetime(2023, 12, 24, 0, 57, 0, 0)
+    end_time = start_time + dt.timedelta(seconds=60)
+    bk_plot = BlankPlot(start_time, bkgmap=True, xlim=[-103.5, -99.5], ylim=[31.5, 35.5], zlim=[0, 20], tlim=[start_time, end_time], title='XLMA Test Plot')
+    dataset = xr.open_dataset('tests/truth/lma_netcdf/lma.nc')
+    times = pd.Series(dataset.event_time.data.flatten())
+    vmin, vmax, colors = color_by_time(pd.to_datetime(times), (start_time, end_time))
+    plot_points(bk_plot, dataset.event_longitude.data, dataset.event_latitude.data, dataset.event_altitude.data/1000,
+                dataset.event_time.data, plot_cmap='rainbow', plot_s=5, plot_vmin=vmin, plot_vmax=vmax,
+                plot_c=colors, edge_color='k', edge_width=0.1, add_to_histogram=True)
+    return bk_plot.fig
+
+@pytest.mark.mpl_image_compare
+def test_plot_feature_plot_points_new_kw():
+    start_time = dt.datetime(2023, 12, 24, 0, 57, 0, 0)
+    end_time = start_time + dt.timedelta(seconds=60)
+    bk_plot = BlankPlot(start_time, bkgmap=True, xlim=[-103.5, -99.5], ylim=[31.5, 35.5], zlim=[0, 20], tlim=[start_time, end_time], title='XLMA Test Plot')
+    dataset = xr.open_dataset('tests/truth/lma_netcdf/lma.nc')
+    times = pd.Series(dataset.event_time.data.flatten())
+    vmin, vmax, colors = color_by_time(pd.to_datetime(times), (start_time, end_time))
+    plot_points(bk_plot, dataset.event_longitude.data, dataset.event_latitude.data, dataset.event_altitude.data/1000,
+                dataset.event_time.data, cmap='rainbow', s=5, vmin=vmin, vmax=vmax,
+                c=colors, edgecolors='k', linewidths=0.1, add_to_histogram=True)
+    return bk_plot.fig
+
+
+@pytest.mark.mpl_image_compare
+def test_plot_feature_plot_points_new_kw_no_bkmap():
+    start_time = dt.datetime(2023, 12, 24, 0, 57, 0, 0)
+    end_time = start_time + dt.timedelta(seconds=60)
+    bk_plot = BlankPlot(start_time, bkgmap=False, xlim=[-103.5, -99.5], ylim=[31.5, 35.5], zlim=[0, 20], tlim=[start_time, end_time], title='XLMA Test Plot')
+    dataset = xr.open_dataset('tests/truth/lma_netcdf/lma.nc')
+    times = pd.Series(dataset.event_time.data.flatten())
+    vmin, vmax, colors = color_by_time(pd.to_datetime(times), (start_time, end_time))
+    plot_points(bk_plot, dataset.event_longitude.data, dataset.event_latitude.data, dataset.event_altitude.data/1000,
+                dataset.event_time.data, cmap='rainbow', s=5, vmin=vmin, vmax=vmax,
+                c=colors, edgecolors='k', linewidths=0.1, add_to_histogram=True)
+    return bk_plot.fig
+
+
+@pytest.mark.mpl_image_compare
+def test_plot_feature_plot_3d_grid():
+    dataset = xr.open_dataset('tests/truth/lma_netcdf/lma.nc')
+    x_edges = np.linspace(-103.5, -99.5, 100)
+    y_edges = np.linspace(31.5, 35.5, 100)
+    z_edges = np.linspace(0, 20, 100)
+    t_edges = md.date2num(pd.date_range(start='2023-12-24T00:57:00', end='2023-12-24T00:58:00', periods=7).values)
+    histograms = setup_hist(dataset.event_longitude.data, dataset.event_latitude.data, dataset.event_altitude.data/1000,
+                            dataset.event_time.data, x_edges, y_edges, z_edges,  t_edges)
+    start_time = dt.datetime(2023, 12, 24, 0, 57, 0, 0)
+    end_time = start_time + dt.timedelta(seconds=60)
+    bk_plot = BlankPlot(start_time, bkgmap=True, xlim=[-103.5, -99.5], ylim=[31.5, 35.5], zlim=[0, 20], tlim=[start_time, end_time], title='XLMA Test Plot')
+
+    plot_3d_grid(bk_plot, x_edges, y_edges, z_edges, t_edges, *histograms, dataset.event_altitude.data/1000, cmap='plasma')
+    return bk_plot.fig
+
+@pytest.mark.mpl_image_compare
+def test_plot_feature_plot_3d_grid_old_kw():
+    dataset = xr.open_dataset('tests/truth/lma_netcdf/lma.nc')
+    x_edges = np.linspace(-103.5, -99.5, 100)
+    y_edges = np.linspace(31.5, 35.5, 100)
+    z_edges = np.linspace(0, 20, 100)
+    t_edges = md.date2num(pd.date_range(start='2023-12-24T00:57:00', end='2023-12-24T00:58:00', periods=7).values)
+    histograms = setup_hist(dataset.event_longitude.data, dataset.event_latitude.data, dataset.event_altitude.data/1000,
+                            dataset.event_time.data, x_edges, y_edges, z_edges,  t_edges)
+    start_time = dt.datetime(2023, 12, 24, 0, 57, 0, 0)
+    end_time = start_time + dt.timedelta(seconds=60)
+    bk_plot = BlankPlot(start_time, bkgmap=True, xlim=[-103.5, -99.5], ylim=[31.5, 35.5], zlim=[0, 20], tlim=[start_time, end_time], title='XLMA Test Plot')
+
+    plot_3d_grid(bk_plot, x_edges, y_edges, z_edges, t_edges, *histograms, dataset.event_altitude.data/1000, plot_cmap='plasma')
+    return bk_plot.fig
+
+
+@pytest.mark.mpl_image_compare
+def test_plot_feature_inset_view():
+    start_time = dt.datetime(2023, 12, 24, 0, 57, 0, 0)
+    end_time = start_time + dt.timedelta(seconds=60)
+    bk_plot = BlankPlot(start_time, bkgmap=True, xlim=[-103.5, -99.5], ylim=[31.5, 35.5], zlim=[0, 20], tlim=[start_time, end_time], title='XLMA Test Plot')
+    dataset = xr.open_dataset('tests/truth/lma_netcdf/lma.nc')
+    times = pd.Series(dataset.event_time.data.flatten())
+    vmin, vmax, colors = color_by_time(pd.to_datetime(times), (start_time, end_time))
+    plot_points(bk_plot, dataset.event_longitude.data, dataset.event_latitude.data, dataset.event_altitude.data/1000,
+                dataset.event_time.data, cmap='rainbow', s=5, vmin=vmin, vmax=vmax,
+                c=colors, edgecolors='k', linewidths=0.1, add_to_histogram=True)
+    inset_view(bk_plot, dataset.event_longitude.data, dataset.event_latitude.data,
+               [-102.75, -102.25], [32, 32.5], .01, .01)
+    return bk_plot.fig