From 173688163294b89a2bfa278b66bbcd012853b6c8 Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Tue, 10 Oct 2023 07:50:25 -0500 Subject: [PATCH 1/9] add matrix testing --- .github/workflows/matrix_ci.yml | 51 +++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/matrix_ci.yml diff --git a/.github/workflows/matrix_ci.yml b/.github/workflows/matrix_ci.yml new file mode 100644 index 00000000..fc7d8d47 --- /dev/null +++ b/.github/workflows/matrix_ci.yml @@ -0,0 +1,51 @@ +name: Matrix Testing CI +# this is similar to the pyart CI action +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# This job installs dependencies and runs tests across a matrix of python and OS versions. +jobs: + build: + name: ${{ matrix.os }}-${{ matrix.python-version }} + if: github.repository == 'tobac-project/tobac' + runs-on: ${{ matrix.os }}-latest + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + os: [macos, ubuntu, windows] + + steps: + - uses: actions/checkout@v2 + + # Install micromamba and dependencies + - name: Setup Conda Environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: continuous_integration/environment-ci.yml + activate-environment: pyart-dev + cache-downloads: true + channels: conda-forge + channel-priority: strict + python-version: ${{ matrix.python-version }} + + - name: Fetch all history for all tags and branches + run: | + git fetch --prune --unshallow + + - name: Install tobac + shell: bash -l {0} + run: | + python -m pip install -e . --no-deps --force-reinstall + + - name: Run Tests + id: run_tests + shell: bash -l {0} + run: | + python -m pytest -v --cov=./ --cov-report=xml From 7da56d8e329a50edb9af49aedc6029d1b05db894 Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Tue, 10 Oct 2023 07:51:56 -0500 Subject: [PATCH 2/9] update matrix testing to run in forks --- .github/workflows/matrix_ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/matrix_ci.yml b/.github/workflows/matrix_ci.yml index fc7d8d47..f83cb41c 100644 --- a/.github/workflows/matrix_ci.yml +++ b/.github/workflows/matrix_ci.yml @@ -7,10 +7,11 @@ concurrency: cancel-in-progress: true # This job installs dependencies and runs tests across a matrix of python and OS versions. +#Add if: github.repository == 'tobac-project/tobac' to limit runs to tobac repo + jobs: build: name: ${{ matrix.os }}-${{ matrix.python-version }} - if: github.repository == 'tobac-project/tobac' runs-on: ${{ matrix.os }}-latest defaults: run: From 295c089415e972506009947b28353852a3a108cf Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Tue, 10 Oct 2023 07:55:17 -0500 Subject: [PATCH 3/9] add environment for matrix CI --- .github/workflows/matrix_ci.yml | 2 +- environment-ci.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 environment-ci.yml diff --git a/.github/workflows/matrix_ci.yml b/.github/workflows/matrix_ci.yml index f83cb41c..75c23d05 100644 --- a/.github/workflows/matrix_ci.yml +++ b/.github/workflows/matrix_ci.yml @@ -29,7 +29,7 @@ jobs: - name: Setup Conda Environment uses: mamba-org/setup-micromamba@v1 with: - environment-file: continuous_integration/environment-ci.yml + environment-file: environment-ci.yml activate-environment: pyart-dev cache-downloads: true channels: conda-forge diff --git a/environment-ci.yml b/environment-ci.yml new file mode 100644 index 00000000..b53f5929 --- /dev/null +++ b/environment-ci.yml @@ -0,0 +1,14 @@ +name: pyart-dev +channels: + - conda-forge +dependencies: + - numpy + - scipy + - scikit-image + - scikit-learn + - pandas + - matplotlib + - iris + - xarray + - cartopy + - trackpy From da9aa4b61f2d2790a544f909e3a59e2c903e168c Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Tue, 10 Oct 2023 07:57:09 -0500 Subject: [PATCH 4/9] add pytest to environment for CI --- environment-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment-ci.yml b/environment-ci.yml index b53f5929..af4fc715 100644 --- a/environment-ci.yml +++ b/environment-ci.yml @@ -12,3 +12,4 @@ dependencies: - xarray - cartopy - trackpy + - pytest From e2cc82b30eedc16de5e2504af2f3a0625c602a09 Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Tue, 10 Oct 2023 08:00:32 -0500 Subject: [PATCH 5/9] update to turn off code coverage --- .github/workflows/matrix_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/matrix_ci.yml b/.github/workflows/matrix_ci.yml index 75c23d05..5232da64 100644 --- a/.github/workflows/matrix_ci.yml +++ b/.github/workflows/matrix_ci.yml @@ -49,4 +49,4 @@ jobs: id: run_tests shell: bash -l {0} run: | - python -m pytest -v --cov=./ --cov-report=xml + python -m pytest -v From 2f5c6da979989e88b61abc2a3982f3328f6feabc Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Tue, 10 Oct 2023 12:24:39 -0500 Subject: [PATCH 6/9] limit to running on main tobac repo only --- .github/workflows/matrix_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/matrix_ci.yml b/.github/workflows/matrix_ci.yml index 5232da64..2da6b38f 100644 --- a/.github/workflows/matrix_ci.yml +++ b/.github/workflows/matrix_ci.yml @@ -13,6 +13,7 @@ jobs: build: name: ${{ matrix.os }}-${{ matrix.python-version }} runs-on: ${{ matrix.os }}-latest + if: github.repository == 'tobac-project/tobac' defaults: run: shell: bash -l {0} From b0f36c0df2d56756826a99ef387c1ba7646e2ca8 Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Wed, 11 Oct 2023 08:24:06 -0500 Subject: [PATCH 7/9] preserve data type for feature and frame --- tobac/utils/general.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tobac/utils/general.py b/tobac/utils/general.py index 7836a9c6..40de28ef 100644 --- a/tobac/utils/general.py +++ b/tobac/utils/general.py @@ -623,10 +623,16 @@ def combine_feature_dataframes( if old_feature_column_name is not None: combined_df[old_feature_column_name] = combined_df["feature"] # count_per_time = combined_feats.groupby('time')['index'].count() - combined_df["frame"] = combined_df["time"].rank(method="dense").astype(int) - 1 + original_frame_dtype = combined_df["frame"].dtype + combined_df["frame"] = ( + combined_df["time"].rank(method="dense").astype(original_frame_dtype) - 1 + ) combined_sorted = combined_df.sort_values(sort_features_by, ignore_index=True) if renumber_features: - combined_sorted["feature"] = np.arange(1, len(combined_sorted) + 1) + original_feature_dtype = combined_df["feature"].dtype + combined_sorted["feature"] = np.arange( + 1, len(combined_sorted) + 1, dtype=original_feature_dtype + ) combined_sorted = combined_sorted.reset_index(drop=True) return combined_sorted From 7986c8a3975c11bf26795b2b0217df4ee1216fef Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 9 Nov 2023 22:15:23 -0600 Subject: [PATCH 8/9] update to integer division for times --- tobac/testing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tobac/testing.py b/tobac/testing.py index 6eb38d50..8614f176 100644 --- a/tobac/testing.py +++ b/tobac/testing.py @@ -62,7 +62,7 @@ def make_simple_sample_data_2D(data_type="iris"): ) * np.exp(-np.power(yy - y_i, 2.0) / (2 * np.power(10e3, 2.0))) t_start = datetime.datetime(1970, 1, 1, 0, 0) - t_points = (t - t_start).astype("timedelta64[ms]").astype(int) / 1000 + t_points = (t - t_start).astype("timedelta64[ms]").astype(int) // 1000 t_coord = DimCoord( t_points, standard_name="time", @@ -176,7 +176,7 @@ def make_sample_data_2D_3blobs(data_type="iris"): -np.power(xx - x_i, 2.0) / (2 * np.power(10e3, 2.0)) ) * np.exp(-np.power(yy - y_i, 2.0) / (2 * np.power(10e3, 2.0))) t_start = datetime.datetime(1970, 1, 1, 0, 0) - t_points = (t - t_start).astype("timedelta64[ms]").astype(int) / 1000 + t_points = (t - t_start).astype("timedelta64[ms]").astype(int) // 1000 t_coord = DimCoord( t_points, standard_name="time", @@ -286,7 +286,7 @@ def make_sample_data_2D_3blobs_inv(data_type="iris"): ) * np.exp(-np.power(yy - y_i, 2.0) / (2 * np.power(10e3, 2.0))) t_start = datetime.datetime(1970, 1, 1, 0, 0) - t_points = (t - t_start).astype("timedelta64[ms]").astype(int) / 1000 + t_points = (t - t_start).astype("timedelta64[ms]").astype(int) // 1000 t_coord = DimCoord( t_points, @@ -432,7 +432,7 @@ def make_sample_data_3D_3blobs(data_type="iris", invert_xy=False): ) t_start = datetime.datetime(1970, 1, 1, 0, 0) - t_points = (t - t_start).astype("timedelta64[ms]").astype(int) / 1000 + t_points = (t - t_start).astype("timedelta64[ms]").astype(int) // 1000 t_coord = DimCoord( t_points, standard_name="time", From b7739d58f209510a55b9520bc25b615a68483fdd Mon Sep 17 00:00:00 2001 From: Sean Freeman Date: Thu, 9 Nov 2023 22:21:06 -0600 Subject: [PATCH 9/9] add typing_extensions to various requirements --- dev_requirements.txt | 2 +- environment-ci.yml | 1 + example_requirements.txt | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index f9cf126e..91298230 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -12,4 +12,4 @@ trackpy pre-commit black pytest - +typing_extensions diff --git a/environment-ci.yml b/environment-ci.yml index af4fc715..4f0b954b 100644 --- a/environment-ci.yml +++ b/environment-ci.yml @@ -13,3 +13,4 @@ dependencies: - cartopy - trackpy - pytest + - typing_extensions diff --git a/example_requirements.txt b/example_requirements.txt index 38bb0945..9ef67557 100644 --- a/example_requirements.txt +++ b/example_requirements.txt @@ -14,4 +14,5 @@ notebook pytables s3fs arm_pyart -h5netcdf \ No newline at end of file +h5netcdf +typing_extensions \ No newline at end of file