Upgrade all files to Python 3.7+ syntax (#486)

* Upgrade all files to Python 3.7+ syntax

* Update contributing guide

* Move pyupgrade to correct section

.github/workflows/style.yaml

@@ -65,3 +65,17 @@ jobs:
       run: pip install .[style]
     - name: Run pydocstyle checks
       run: pydocstyle
+  pyupgrade:
+    name: pyupgrade
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone repo
+      uses: actions/checkout@v2
+    - name: Set up python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Install pip dependencies
+      run: pip install .[style]
+    - name: Run pyupgrade checks
+      run: pyupgrade --py37-plus $(find . -name "*.py")

.pre-commit-config.yaml

@@ -11,6 +11,12 @@ repos:
         -   id: black
             args: [--skip-magic-trailing-comma]
 
+    -   repo: https://github.com/asottile/pyupgrade
+        rev: 2.31.1
+        hooks:
+        -   id: pyupgrade
+            args: [--py37-plus]
+
     -   repo: https://gitlab.com/pycqa/flake8.git
         rev: 3.9.2
         hooks:

docs/user/contributing.rst

@@ -94,14 +94,16 @@ In order to remain `PEP-8 <https://www.python.org/dev/peps/pep-0008/>`_ complian
 * `isort <https://pycqa.github.io/isort/>`_ for import ordering
 * `flake8 <https://flake8.pycqa.org/>`_ for code formatting
 * `pydocstyle <https://www.pydocstyle.org/>`_ for docstrings
+* `pyupgrade <https://github.com/asottile/pyupgrade>`_ for code formatting
 * `mypy <https://mypy.readthedocs.io/>`_ for static type analysis
 
-All of these tools should be used from the root of the project to ensure that our configuration files are found. Black and isort are relatively easy to use, and will automatically format your code for you:
+All of these tools should be used from the root of the project to ensure that our configuration files are found. Black, isort, and pyupgrade are relatively easy to use, and will automatically format your code for you:
 
 .. code-block:: console
 
    $ black .
    $ isort .
+   $ pyupgrade --py37-plus $(find . -name "*.py")
 
 
 Flake8, pydocstyle, and mypy won't format your code for you, but they will warn you about potential issues with your code or docstrings:

setup.cfg

@@ -100,6 +100,8 @@ style =
     isort[colors]>=5.8
     # pydocstyle 6.1+ required for pyproject.toml support
     pydocstyle[toml]>=6.1
+    # pyupgrade 1.24+ required for --py37-plus flag
+    pyupgrade>=1.24
 # Optional testing requirements
 tests =
     # mypy 0.900+ required for pyproject.toml support

tests/data/globbiomass/data.py

@@ -45,7 +45,7 @@ def create_file(path: str, dtype: str, num_channels: int) -> None:
 if __name__ == "__main__":
 
     for measurement, file_paths in files.items():
-        zipfilename = "N00E020_{}.zip".format(measurement)
+        zipfilename = f"N00E020_{measurement}.zip"
         files_to_zip = []
         for path in file_paths:
             # remove old data

tests/data/openbuildings/data.py

@@ -34,7 +34,7 @@ def create_meta_data_file(zipfilename):
                 },
                 "properties": {
                     "tile_id": "025",
-                    "tile_url": "polygons_s2_level_4_gzip/{}".format(zipfilename),
+                    "tile_url": f"polygons_s2_level_4_gzip/{zipfilename}",
                     "size_mb": 0.2,
                 },
             }

tests/datasets/test_openbuildings.py

@@ -107,7 +107,7 @@ def test_no_meta_data_found(self, tmp_path: Path) -> None:
 
     def test_nothing_in_index(self, dataset: OpenBuildings, tmp_path: Path) -> None:
         # change meta data to another 'title_url' so that there is no match found
-        with open(os.path.join(tmp_path, "tiles.geojson"), "r") as f:
+        with open(os.path.join(tmp_path, "tiles.geojson")) as f:
             content = json.load(f)
             content["features"][0]["properties"]["tile_url"] = "mismatch.csv.gz"
 

tests/test_train.py

@@ -14,7 +14,7 @@
 
 def test_required_args() -> None:
     args = [sys.executable, "train.py"]
-    ps = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    ps = subprocess.run(args, capture_output=True)
     assert ps.returncode != 0
     assert b"ConfigKeyError" in ps.stderr
 
@@ -29,7 +29,7 @@ def test_output_file(tmp_path: Path) -> None:
         "program.output_dir=" + str(output_file),
         "experiment.task=test",
     ]
-    ps = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    ps = subprocess.run(args, capture_output=True)
     assert ps.returncode != 0
     assert b"NotADirectoryError" in ps.stderr
 
@@ -47,7 +47,7 @@ def test_experiment_dir_not_empty(tmp_path: Path) -> None:
         "program.output_dir=" + str(output_dir),
         "experiment.task=test",
     ]
-    ps = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    ps = subprocess.run(args, capture_output=True)
     assert ps.returncode != 0
     assert b"FileExistsError" in ps.stderr
 
@@ -72,9 +72,7 @@ def test_overwrite_experiment_dir(tmp_path: Path) -> None:
         "program.overwrite=True",
         "trainer.fast_dev_run=1",
     ]
-    ps = subprocess.run(
-        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
-    )
+    ps = subprocess.run(args, capture_output=True, check=True)
     assert re.search(
         b"The experiment directory, .*, already exists, we might overwrite data in it!",
         ps.stdout,
@@ -90,7 +88,7 @@ def test_invalid_task(tmp_path: Path) -> None:
         "program.output_dir=" + str(output_dir),
         "experiment.task=foo",
     ]
-    ps = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    ps = subprocess.run(args, capture_output=True)
     assert ps.returncode != 0
     assert b"ValueError" in ps.stderr
 
@@ -106,7 +104,7 @@ def test_missing_config_file(tmp_path: Path) -> None:
         "experiment.task=test",
         "config_file=" + str(config_file),
     ]
-    ps = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    ps = subprocess.run(args, capture_output=True)
     assert ps.returncode != 0
     assert b"FileNotFoundError" in ps.stderr
 

torchgeo/datasets/advance.py

@@ -117,7 +117,7 @@ def __init__(
             )
 
         self.files = self._load_files(self.root)
-        self.classes = sorted(set(f["cls"] for f in self.files))
+        self.classes = sorted({f["cls"] for f in self.files})
         self.class_to_idx: Dict[str, int] = {c: i for i, c in enumerate(self.classes)}
 
     def __getitem__(self, index: int) -> Dict[str, Tensor]:

torchgeo/datasets/agb_live_woody_density.py

@@ -114,7 +114,7 @@ def _download(self) -> None:
         """Download the dataset."""
         download_url(self.url, self.root, self.base_filename)
 
-        with open(os.path.join(self.root, self.base_filename), "r") as f:
+        with open(os.path.join(self.root, self.base_filename)) as f:
             content = json.load(f)
 
         for item in content["features"]:

torchgeo/datasets/benin_cashews.py

@@ -365,7 +365,7 @@ def _load_mask(self, transform: rasterio.Affine) -> Tensor:
         mask_geojson_fn = os.path.join(
             self.root, "ts_cashew_benin_labels", "_common", "labels.geojson"
         )
-        with open(mask_geojson_fn, "r") as f:
+        with open(mask_geojson_fn) as f:
             geojson = json.load(f)
 
         labels = [

torchgeo/datasets/bigearthnet.py

@@ -412,7 +412,7 @@ def _load_target(self, index: int) -> Tensor:
             folder = self.folders[index]["s1"]
 
         path = glob.glob(os.path.join(folder, "*.json"))[0]
-        with open(path, "r") as f:
+        with open(path) as f:
             labels = json.load(f)["labels"]
 
         # labels -> indices

torchgeo/datasets/cms_mangrove_canopy.py

@@ -215,7 +215,7 @@ def __init__(
         )
         self.measurement = measurement
 
-        self.filename_glob = "**/Mangrove_{}_{}*".format(self.measurement, self.country)
+        self.filename_glob = f"**/Mangrove_{self.measurement}_{self.country}*"
 
         self._verify()
 

torchgeo/datasets/eurosat.py

@@ -146,7 +146,7 @@ def __init__(
         self._verify()
 
         valid_fns = set()
-        with open(os.path.join(self.root, f"eurosat-{split}.txt"), "r") as f:
+        with open(os.path.join(self.root, f"eurosat-{split}.txt")) as f:
             for fn in f:
                 valid_fns.add(fn.strip().replace(".jpg", ".tif"))
         is_in_split: Callable[[str], bool] = lambda x: os.path.basename(x) in valid_fns

torchgeo/datasets/globbiomass.py

@@ -155,8 +155,8 @@ def __init__(
         )
         self.measurement = measurement
 
-        self.filename_glob = "*0_{}*.tif".format(self.measurement)
-        self.zipfile_glob = "*0_{}.zip".format(self.measurement)
+        self.filename_glob = f"*0_{self.measurement}*.tif"
+        self.zipfile_glob = f"*0_{self.measurement}.zip"
 
         self._verify()
 

torchgeo/datasets/idtrees.py

@@ -289,10 +289,10 @@ def _load_boxes(self, path: str) -> Tensor:
         with rasterio.open(path) as f:
             for geom in geoms:
                 coords = [f.index(x, y) for x, y in geom]
-                xmin = min([coord[0] for coord in coords])
-                xmax = max([coord[0] for coord in coords])
-                ymin = min([coord[1] for coord in coords])
-                ymax = max([coord[1] for coord in coords])
+                xmin = min(coord[0] for coord in coords)
+                xmax = max(coord[0] for coord in coords)
+                ymin = min(coord[1] for coord in coords)
+                ymax = max(coord[1] for coord in coords)
                 boxes.append([xmin, ymin, xmax, ymax])
 
         tensor = torch.tensor(boxes)

torchgeo/datasets/levircd.py

@@ -137,7 +137,7 @@ def _load_files(
         """
         files = []
         images = glob.glob(os.path.join(root, directory, split, "A", "*.png"))
-        images = sorted([os.path.basename(image) for image in images])
+        images = sorted(os.path.basename(image) for image in images)
         for image in images:
             image1 = os.path.join(root, directory, split, "A", image)
             image2 = os.path.join(root, directory, split, "B", image)

torchgeo/datasets/nasa_marine_debris.py

@@ -150,7 +150,7 @@ def _load_files(self) -> List[Dict[str, str]]:
         image_root = os.path.join(self.root, self.directories[0])
         target_root = os.path.join(self.root, self.directories[1])
         image_folders = sorted(
-            [f for f in os.listdir(image_root) if not f.endswith("json")]
+            f for f in os.listdir(image_root) if not f.endswith("json")
         )
 
         files = []

torchgeo/datasets/openbuildings.py

@@ -402,7 +402,7 @@ def _verify(self) -> None:
         for zipfile in glob.iglob(pathname):
             filename = os.path.basename(zipfile)
             if self.checksum and not check_integrity(zipfile, self.md5s[filename]):
-                raise RuntimeError("Dataset found, but corrupted: {}.".format(filename))
+                raise RuntimeError(f"Dataset found, but corrupted: {filename}.")
             i += 1
 
         if i != 0:

torchgeo/datasets/oscd.py

@@ -174,7 +174,7 @@ def get_image_paths(ind: int) -> List[str]:
 
             with open(os.path.join(images_root, region, "dates.txt")) as f:
                 dates = tuple(
-                    [line.split()[-1] for line in f.read().strip().splitlines()]
+                    line.split()[-1] for line in f.read().strip().splitlines()
                 )
 
             regions.append(

torchgeo/datasets/resisc45.py

@@ -180,7 +180,7 @@ def __init__(
         self._verify()
 
         valid_fns = set()
-        with open(os.path.join(self.root, f"resisc45-{split}.txt"), "r") as f:
+        with open(os.path.join(self.root, f"resisc45-{split}.txt")) as f:
             for fn in f:
                 valid_fns.add(fn.strip())
         is_in_split: Callable[[str], bool] = lambda x: os.path.basename(x) in valid_fns

torchgeo/datasets/sen12ms.py

@@ -261,9 +261,9 @@ def _load_raster(self, filename: str, source: str) -> Tensor:
         with rasterio.open(
             os.path.join(
                 self.root,
-                "{0}_{1}".format(*parts),
+                "{}_{}".format(*parts),
                 "{2}_{3}".format(*parts),
-                "{0}_{1}_{2}_{3}_{4}".format(*parts),
+                "{}_{}_{}_{}_{}".format(*parts),
             )
         ) as f:
             array = f.read().astype(np.int32)

torchgeo/datasets/ucmerced.py

@@ -132,7 +132,7 @@ def __init__(
         self._verify()
 
         valid_fns = set()
-        with open(os.path.join(self.root, f"uc_merced-{split}.txt"), "r") as f:
+        with open(os.path.join(self.root, f"uc_merced-{split}.txt")) as f:
             for fn in f:
                 valid_fns.add(fn.strip())
         is_in_split: Callable[[str], bool] = lambda x: os.path.basename(x) in valid_fns

torchgeo/datasets/usavars.py

@@ -118,15 +118,10 @@ def __init__(
 
         self.files = self._load_files()
 
-        self.label_dfs = dict(
-            [
-                (
-                    lab,
-                    pd.read_csv(os.path.join(self.root, lab + ".csv"), index_col="ID"),
-                )
-                for lab in self.labels
-            ]
-        )
+        self.label_dfs = {
+            lab: pd.read_csv(os.path.join(self.root, lab + ".csv"), index_col="ID")
+            for lab in self.labels
+        }
 
     def __getitem__(self, index: int) -> Dict[str, Tensor]:
         """Return an index within the dataset.

torchgeo/datasets/utils.py

@@ -166,7 +166,7 @@ def download_and_extract_archive(
     download_url(url, download_root, filename, md5)
 
     archive = os.path.join(download_root, filename)
-    print("Extracting {} to {}".format(archive, extract_root))
+    print(f"Extracting {archive} to {extract_root}")
     extract_archive(archive, extract_root)
 
 

torchgeo/models/fcsiam.py

@@ -92,7 +92,7 @@ def __init__(
             kernel_size=3,
         )
         self.classification_head = None
-        self.name = "u-{}".format(encoder_name)
+        self.name = f"u-{encoder_name}"
         self.initialize()
 
     def forward(self, x: Tensor) -> Tensor: