-
Notifications
You must be signed in to change notification settings - Fork 5
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add first iteration of tests to repo #42
Changes from all commits
a261f32
e401606
0ba4f63
557755a
02ecd27
33d3900
f33b994
6ef2d5b
c463aa8
74f7923
e65d362
70229b1
0cd0ec8
dfc70d3
021a229
c1e0236
9f5658b
4754167
b867d6f
f5fb2a6
81ae4fc
505e36f
ade3b23
5f63063
eda9b7e
47b9bcc
cd184fd
2dae7ab
3676b4e
1d6ae75
a5bbbb1
83b5d2c
c9cf866
43ffbbc
ecb5224
719817f
53e0081
5556fc5
6d37c83
fb64d14
6bcd20a
4e34de0
69501a0
1eb13aa
1f0c268
bf437ac
53ef7d1
05cafad
ab88714
06ac857
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
name: DEA Intertidal Image Push | ||
name: Image build and test | ||
|
||
env: | ||
IMAGE_NAME: geoscienceaustralia/dea-intertidal | ||
|
@@ -11,11 +11,13 @@ on: | |
paths: | ||
- 'intertidal/**' | ||
- 'data/**' | ||
- 'tests/**' | ||
- '.github/workflows/dea-intertidal-image.yml' | ||
- 'Dockerfile' | ||
- 'requirements.in' | ||
- 'setup.py' | ||
- 'docker-compose.yml' | ||
- 'codecov.yaml' | ||
|
||
release: | ||
types: [created, edited, published] | ||
|
@@ -31,13 +33,33 @@ jobs: | |
steps: | ||
- name: Checkout code | ||
uses: actions/checkout@v3 | ||
with: | ||
fetch-depth: 0 | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe makes the git clone faster? |
||
- name: Build DEA Intertidal image | ||
timeout-minutes: 20 | ||
shell: bash | ||
run: | | ||
docker-compose build | ||
|
||
- name: Run tests | ||
run: | | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This downloads tide data from Dropbox (to replace with S3 access to tide modelling files), then runs some tests using |
||
# Download tide modelling files and unzip | ||
# TODO: Replace with S3 sync from dea-non-public-data | ||
wget --no-verbose https://www.dropbox.com/s/uemd8ib2vfw5nad/tide_models.zip?dl=1 -O tide_models.zip | ||
unzip -q tide_models.zip | ||
|
||
# Run integration tests using Docker | ||
docker compose run dea_intertidal pytest -v --cov=intertidal --cov-report=xml tests | ||
|
||
- name: Upload coverage reports to Codecov | ||
uses: codecov/codecov-action@v3 | ||
env: | ||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | ||
|
||
# - name: Setup upterm session | ||
# uses: lhotari/action-upterm@v1 | ||
|
||
push_ecr: | ||
needs: [test] | ||
runs-on: ubuntu-latest | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
ignore: | ||
- "intertidal/validation.py" | ||
- "intertidal/composites.py" | ||
|
||
coverage: | ||
precision: 1 | ||
round: down | ||
range: "50...90" |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,4 +6,8 @@ | |
!*.ipynb | ||
!*.sh | ||
!*.yaml | ||
!*.gitignore | ||
!*.yml | ||
!*.in | ||
!*.gitignore | ||
!*.dockerignore | ||
!*Dockerfile |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -820,7 +820,7 @@ def pixel_dem(interval_ds, ndwi_thresh=0.1, interp_intervals=200, smooth_radius= | |
smoothed_ds = interval_ds.rolling( | ||
interval=smooth_radius, | ||
center=False, | ||
min_periods=1, #int(smooth_radius / 2.0), | ||
min_periods=1, # int(smooth_radius / 2.0), | ||
).mean() | ||
else: | ||
smoothed_ds = interval_ds | ||
|
@@ -844,6 +844,7 @@ def pixel_dem(interval_ds, ndwi_thresh=0.1, interp_intervals=200, smooth_radius= | |
# Export as xr.Dataset | ||
return dem_flat.to_dataset(name="elevation") | ||
|
||
|
||
def pixel_dem_debug( | ||
x, | ||
y, | ||
|
@@ -1113,7 +1114,7 @@ def clean_edge_pixels(ds): | |
effects mean that modelled elevations are likely to be inaccurate. | ||
|
||
This function uses binary dilation to identify the edges of | ||
intertidal elevation data with greater than 0 elevation. The | ||
intertidal elevation data with greater than 0 elevation. The | ||
resulting mask is applied to the elevation dataset to remove upper | ||
intertidal edge pixels from both elevation and uncertainty datasets. | ||
|
||
|
@@ -1247,7 +1248,7 @@ def elevation( | |
ancillary_points="data/raw/tide_correlations_2017-2019.geojson", | ||
# ancillary_points="data/raw/tide_correlation_points_spearmanndwi_nt.geojson", | ||
top_n=3, | ||
reduce_method='mean', | ||
reduce_method="mean", | ||
resolution=3000, | ||
) | ||
|
||
|
@@ -1347,14 +1348,14 @@ def elevation( | |
@click.option( | ||
"--start_date", | ||
type=str, | ||
default="2020", | ||
default="2019", | ||
help="The start date of satellite data to load from the " | ||
"datacube. This can be any date format accepted by datacube. ", | ||
) | ||
@click.option( | ||
"--end_date", | ||
type=str, | ||
default="2022", | ||
default="2021", | ||
help="The end date of satellite data to load from the " | ||
"datacube. This can be any date format accepted by datacube. ", | ||
) | ||
|
@@ -1494,44 +1495,55 @@ def intertidal_cli( | |
configure_s3_access(cloud_defaults=True, aws_unsigned=aws_unsigned) | ||
|
||
# Create output folder. If it doesn't exist, create it | ||
output_dir = f"data/interim/{study_area}/{start_date}-{end_date}-spearman" | ||
output_dir = f"data/interim/{study_area}/{start_date}-{end_date}" | ||
os.makedirs(output_dir, exist_ok=True) | ||
|
||
try: | ||
log.info(f"Study area {study_area}: Loading satellite data") | ||
|
||
# Connect to datacube to load data | ||
dc = datacube.Datacube(app="Intertidal_CLI") | ||
|
||
|
||
# Create local dask cluster to improve data load time | ||
client = create_local_dask_cluster(return_client=True) | ||
|
||
satellite_ds = load_data( | ||
dc=dc, | ||
study_area=study_area, | ||
time_range=(start_date, end_date), | ||
resolution=resolution, | ||
crs="EPSG:3577", | ||
include_s2=True, | ||
include_ls=True, | ||
filter_gqa=True, | ||
max_cloudcover=90, | ||
skip_broken_datasets=True, | ||
) | ||
if study_area == "testing": | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use pickle file instead of loading data from datacube if study area == "testing" |
||
log.info(f"Running in testing mode") | ||
import pickle | ||
|
||
# Load data | ||
satellite_ds.load() | ||
with open("tests/data/satellite_ds.pickle", "rb") as handle: | ||
satellite_ds = pickle.load(handle) | ||
valid_mask = None | ||
|
||
# Load data from GA's Australian Bathymetry and Topography Grid 2009 | ||
topobathy_ds = load_topobathy( | ||
dc, satellite_ds, product="ga_multi_ausbath_0", resampling="bilinear" | ||
) | ||
else: | ||
|
||
# Connect to datacube to load data | ||
dc = datacube.Datacube(app="Intertidal_CLI") | ||
|
||
satellite_ds = load_data( | ||
dc=dc, | ||
study_area=study_area, | ||
time_range=(start_date, end_date), | ||
resolution=resolution, | ||
crs="EPSG:3577", | ||
include_s2=True, | ||
include_ls=True, | ||
filter_gqa=True, | ||
max_cloudcover=90, | ||
skip_broken_datasets=True, | ||
) | ||
|
||
# Load data | ||
satellite_ds.load() | ||
|
||
# Load data from GA's Australian Bathymetry and Topography Grid 2009 | ||
topobathy_ds = load_topobathy( | ||
dc, satellite_ds, product="ga_multi_ausbath_0", resampling="bilinear" | ||
) | ||
valid_mask = topobathy_ds.height_depth > -20 | ||
|
||
# Calculate elevation | ||
log.info(f"Study area {study_area}: Calculating Intertidal Elevation") | ||
ds, ds_aux, tide_m = elevation( | ||
satellite_ds, | ||
valid_mask=topobathy_ds.height_depth > -20, | ||
valid_mask=valid_mask, | ||
ndwi_thresh=ndwi_thresh, | ||
min_freq=min_freq, | ||
max_freq=max_freq, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -14,8 +14,6 @@ | |
# from pyproj import Transformer | ||
# from scipy.signal import argrelmax, argrelmin | ||
|
||
from scipy.interpolate import interp1d | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not used below; removing for now |
||
|
||
from dea_tools.coastal import pixel_tides, model_tides | ||
from intertidal.tide_modelling import pixel_tides_ensemble | ||
# from intertidal.elevation import pixel_tides_ensemble | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -77,7 +77,7 @@ def extents( | |
freq, | ||
dem, | ||
corr, | ||
land_use_mask="/gdata1/data/land_use/ABARES_CLUM/geotiff_clum_50m1220m/clum_50m1220m.tif", | ||
land_use_mask="https://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/abares_clum_2020/clum_50m1220m.tiff", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Load data from S3 so it works in the tests |
||
): | ||
""" | ||
Classify coastal ecosystems into broad classes based | ||
|
@@ -147,9 +147,6 @@ def extents( | |
crosses zero) | ||
|
||
""" | ||
## Connect to datacube to load `ocean_da` | ||
dc = datacube.Datacube(app="ocean_masking") | ||
|
||
# Load the land use dataset to mask out misclassified extents classes caused by urban land class | ||
landuse_da = load_reproject( | ||
path=land_use_mask, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -14,7 +14,11 @@ pyTMD==2.0.5 | |
pandas==1.5.3 | ||
pygeos==0.14 | ||
pyproj==3.4.1 | ||
pytest | ||
pytest-dependency | ||
pytest-cov | ||
pytz==2023.3 | ||
rioxarray | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Had to add back |
||
rasterio==1.3.4 | ||
seaborn==0.13.0 | ||
scikit_image==0.19.3 | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
import pytest | ||
from click.testing import CliRunner | ||
from intertidal.elevation import intertidal_cli, elevation | ||
import pickle | ||
|
||
|
||
@pytest.fixture() | ||
def satellite_ds(): | ||
with open("tests/data/satellite_ds.pickle", "rb") as handle: | ||
return pickle.load(handle) | ||
|
||
|
||
def test_intertidal_cli(): | ||
runner = CliRunner() | ||
result = runner.invoke( | ||
intertidal_cli, | ||
[ | ||
"--study_area", | ||
"testing", | ||
"--start_date", | ||
"2020", | ||
"--end_date", | ||
"2022", | ||
"--modelled_freq", | ||
"3h", | ||
], | ||
) | ||
assert result.exit_code == 0 | ||
|
||
|
||
def test_elevation(satellite_ds): | ||
ds, ds_aux, tide_m = elevation( | ||
satellite_ds, | ||
valid_mask=None, | ||
ndwi_thresh=0.1, | ||
min_freq=0.01, | ||
max_freq=0.99, | ||
min_correlation=0.15, | ||
windows_n=20, | ||
window_prop_tide=0.15, | ||
max_workers=None, | ||
tide_model="FES2014", | ||
tide_model_dir="/var/share/tide_models", | ||
study_area=None, | ||
log=None, | ||
) | ||
|
||
# Verify that ds contains correct variables | ||
assert "elevation" in ds.data_vars | ||
assert "elevation_uncertainty" in ds.data_vars | ||
|
||
# Verify that ds is a single layer | ||
assert "time" not in ds.dims |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Renamed so it shows up nicely on the readme badges