Rwanda Field Boundary: radiant mlhub -> source cooperative

tests/data/rwanda_field_boundary/data.py

@@ -3,99 +3,46 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-import hashlib
 import os
-import shutil
 
 import numpy as np
 import rasterio
+from rasterio.crs import CRS
+from rasterio.transform import Affine
 
 dates = ('2021_03', '2021_04', '2021_08', '2021_10', '2021_11', '2021_12')
 all_bands = ('B01', 'B02', 'B03', 'B04')
 
 SIZE = 32
-NUM_SAMPLES = 5
+DTYPE = np.uint16
+NUM_SAMPLES = 1
 np.random.seed(0)
 
-
-def create_mask(fn: str) -> None:
-    profile = {
-        'driver': 'GTiff',
-        'dtype': 'uint8',
-        'nodata': 0.0,
-        'width': SIZE,
-        'height': SIZE,
-        'count': 1,
-        'crs': 'epsg:3857',
-        'compress': 'lzw',
-        'predictor': 2,
-        'transform': rasterio.Affine(10.0, 0.0, 0.0, 0.0, -10.0, 0.0),
-        'blockysize': 32,
-        'tiled': False,
-        'interleave': 'band',
-    }
-    with rasterio.open(fn, 'w', **profile) as f:
-        f.write(np.random.randint(0, 2, size=(SIZE, SIZE), dtype=np.uint8), 1)
-
-
-def create_img(fn: str) -> None:
-    profile = {
-        'driver': 'GTiff',
-        'dtype': 'uint16',
-        'nodata': 0.0,
-        'width': SIZE,
-        'height': SIZE,
-        'count': 1,
-        'crs': 'epsg:3857',
-        'compress': 'lzw',
-        'predictor': 2,
-        'blockysize': 16,
-        'transform': rasterio.Affine(10.0, 0.0, 0.0, 0.0, -10.0, 0.0),
-        'tiled': False,
-        'interleave': 'band',
-    }
-    with rasterio.open(fn, 'w', **profile) as f:
-        f.write(np.random.randint(0, 2, size=(SIZE, SIZE), dtype=np.uint16), 1)
-
-
-if __name__ == '__main__':
-    # Train and test images
-    for split in ('train', 'test'):
-        for i in range(NUM_SAMPLES):
-            for date in dates:
-                directory = os.path.join(
-                    f'nasa_rwanda_field_boundary_competition_source_{split}',
-                    f'nasa_rwanda_field_boundary_competition_source_{split}_{i:02d}_{date}',  # noqa: E501
-                )
-                os.makedirs(directory, exist_ok=True)
-                for band in all_bands:
-                    create_img(os.path.join(directory, f'{band}.tif'))
-
-        # Create collections.json, this isn't used by the dataset but is checked to
-        # exist
-        with open(
-            f'nasa_rwanda_field_boundary_competition_source_{split}/collections.json',
-            'w',
-        ) as f:
-            f.write('Not used')
-
-    # Train labels
-    for i in range(NUM_SAMPLES):
-        directory = os.path.join(
-            'nasa_rwanda_field_boundary_competition_labels_train',
-            f'nasa_rwanda_field_boundary_competition_labels_train_{i:02d}',
-        )
-        os.makedirs(directory, exist_ok=True)
-        create_mask(os.path.join(directory, 'raster_labels.tif'))
-
-    # Create directories and compute checksums
-    for filename in [
-        'nasa_rwanda_field_boundary_competition_source_train',
-        'nasa_rwanda_field_boundary_competition_source_test',
-        'nasa_rwanda_field_boundary_competition_labels_train',
-    ]:
-        shutil.make_archive(filename, 'gztar', '.', filename)
-        # Compute checksums
-        with open(f'{filename}.tar.gz', 'rb') as f:
-            md5 = hashlib.md5(f.read()).hexdigest()
-            print(f'{filename}: {md5}')
+profile = {
+    'driver': 'GTiff',
+    'dtype': DTYPE,
+    'width': SIZE,
+    'height': SIZE,
+    'count': 1,
+    'crs': CRS.from_epsg(3857),
+    'transform': Affine(
+        4.77731426716, 0.0, 3374518.037700199, 0.0, -4.77731426716, -168438.54642526805
+    ),
+}
+Z = np.random.randint(np.iinfo(DTYPE).max, size=(SIZE, SIZE), dtype=DTYPE)
+
+for sample in range(NUM_SAMPLES):
+    for split in ['train', 'test']:
+        for date in dates:
+            path = os.path.join('source', split, date)
+            os.makedirs(path, exist_ok=True)
+            for band in all_bands:
+                file = os.path.join(path, f'{sample:02}_{band}.tif')
+                with rasterio.open(file, 'w', **profile) as src:
+                    src.write(Z, 1)
+
+    path = os.path.join('labels', 'train')
+    os.makedirs(path, exist_ok=True)
+    file = os.path.join(path, f'{sample:02}.tif')
+    with rasterio.open(file, 'w', **profile) as src:
+        src.write(Z, 1)

tests/datasets/test_rwanda_field_boundary.py

@@ -1,9 +1,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-import glob
 import os
-import shutil
 from pathlib import Path
 
 import matplotlib.pyplot as plt
@@ -19,45 +17,26 @@
     RGBBandsMissingError,
     RwandaFieldBoundary,
 )
-
-
-class Collection:
-    def download(self, output_dir: str, **kwargs: str) -> None:
-        glob_path = os.path.join('tests', 'data', 'rwanda_field_boundary', '*.tar.gz')
-        for tarball in glob.iglob(glob_path):
-            shutil.copy(tarball, output_dir)
-
-
-def fetch(dataset_id: str, **kwargs: str) -> Collection:
-    return Collection()
+from torchgeo.datasets.utils import Executable
 
 
 class TestRwandaFieldBoundary:
     @pytest.fixture(params=['train', 'test'])
     def dataset(
-        self, monkeypatch: MonkeyPatch, tmp_path: Path, request: SubRequest
+        self,
+        azcopy: Executable,
+        monkeypatch: MonkeyPatch,
+        tmp_path: Path,
+        request: SubRequest,
     ) -> RwandaFieldBoundary:
-        radiant_mlhub = pytest.importorskip('radiant_mlhub', minversion='0.3')
-        monkeypatch.setattr(radiant_mlhub.Collection, 'fetch', fetch)
-        monkeypatch.setattr(
-            RwandaFieldBoundary, 'number_of_patches_per_split', {'train': 5, 'test': 5}
-        )
-        monkeypatch.setattr(
-            RwandaFieldBoundary,
-            'md5s',
-            {
-                'train_images': 'af9395e2e49deefebb35fa65fa378ba3',
-                'test_images': 'd104bb82323a39e7c3b3b7dd0156f550',
-                'train_labels': '6cceaf16a141cf73179253a783e7d51b',
-            },
-        )
+        url = os.path.join('tests', 'data', 'rwanda_field_boundary')
+        monkeypatch.setattr(RwandaFieldBoundary, 'url', url)
+        monkeypatch.setattr(RwandaFieldBoundary, 'splits', {'train': 1, 'test': 1})
 
         root = str(tmp_path)
         split = request.param
         transforms = nn.Identity()
-        return RwandaFieldBoundary(
-            root, split, transforms=transforms, api_key='', download=True, checksum=True
-        )
+        return RwandaFieldBoundary(root, split, transforms=transforms, download=True)
 
     def test_getitem(self, dataset: RwandaFieldBoundary) -> None:
         x = dataset[0]
@@ -69,23 +48,12 @@ def test_getitem(self, dataset: RwandaFieldBoundary) -> None:
             assert 'mask' not in x
 
     def test_len(self, dataset: RwandaFieldBoundary) -> None:
-        assert len(dataset) == 5
+        assert len(dataset) == 1
 
     def test_add(self, dataset: RwandaFieldBoundary) -> None:
         ds = dataset + dataset
         assert isinstance(ds, ConcatDataset)
-        assert len(ds) == 10
-
-    def test_needs_extraction(self, tmp_path: Path) -> None:
-        root = str(tmp_path)
-        for fn in [
-            'nasa_rwanda_field_boundary_competition_source_train.tar.gz',
-            'nasa_rwanda_field_boundary_competition_source_test.tar.gz',
-            'nasa_rwanda_field_boundary_competition_labels_train.tar.gz',
-        ]:
-            url = os.path.join('tests', 'data', 'rwanda_field_boundary', fn)
-            shutil.copy(url, root)
-        RwandaFieldBoundary(root, checksum=False)
+        assert len(ds) == 2
 
     def test_already_downloaded(self, dataset: RwandaFieldBoundary) -> None:
         RwandaFieldBoundary(root=dataset.root)
@@ -94,35 +62,8 @@ def test_not_downloaded(self, tmp_path: Path) -> None:
         with pytest.raises(DatasetNotFoundError, match='Dataset not found'):
             RwandaFieldBoundary(str(tmp_path))
 
-    def test_corrupted(self, tmp_path: Path) -> None:
-        for fn in [
-            'nasa_rwanda_field_boundary_competition_source_train.tar.gz',
-            'nasa_rwanda_field_boundary_competition_source_test.tar.gz',
-            'nasa_rwanda_field_boundary_competition_labels_train.tar.gz',
-        ]:
-            with open(os.path.join(tmp_path, fn), 'w') as f:
-                f.write('bad')
-        with pytest.raises(RuntimeError, match='Dataset found, but corrupted.'):
-            RwandaFieldBoundary(root=str(tmp_path), checksum=True)
-
-    def test_failed_download(self, monkeypatch: MonkeyPatch, tmp_path: Path) -> None:
-        radiant_mlhub = pytest.importorskip('radiant_mlhub', minversion='0.3')
-        monkeypatch.setattr(radiant_mlhub.Collection, 'fetch', fetch)
-        monkeypatch.setattr(
-            RwandaFieldBoundary,
-            'md5s',
-            {'train_images': 'bad', 'test_images': 'bad', 'train_labels': 'bad'},
-        )
-        root = str(tmp_path)
-        with pytest.raises(RuntimeError, match='Dataset not found or corrupted.'):
-            RwandaFieldBoundary(root, 'train', api_key='', download=True, checksum=True)
-
-    def test_no_api_key(self, tmp_path: Path) -> None:
-        with pytest.raises(RuntimeError, match='Must provide an API key to download'):
-            RwandaFieldBoundary(str(tmp_path), api_key=None, download=True)
-
     def test_invalid_bands(self) -> None:
-        with pytest.raises(ValueError, match='is an invalid band name.'):
+        with pytest.raises(AssertionError):
             RwandaFieldBoundary(bands=('foo', 'bar'))
 
     def test_plot(self, dataset: RwandaFieldBoundary) -> None: