Skip downloading when not in pre-sampled coords (#1821)

* Skip downloading an entry if it is not in the given pre-sampled list. * Add warning message when skipping --------- Co-authored-by: Michiaki Tatsubori <[email protected]> Co-authored-by: Adam J. Stewart <[email protected]>
microsoft · Mar 2, 2024 · d718b24 · d718b24
1 parent 1075f96
commit d718b24
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/experiments/ssl4eo/download_ssl4eo.py b/experiments/ssl4eo/download_ssl4eo.py
@@ -52,6 +52,7 @@
 import json
 import os
 import time
+import warnings
 from collections import defaultdict
 from datetime import date, timedelta
 from multiprocessing.dummy import Lock, Pool
@@ -473,9 +474,15 @@ def update(self, delta: int = 1) -> int:
     counter = Counter()
 
     def worker(idx: int) -> None:
+        # Skip if idx has already been downloaded
         if idx in ext_coords.keys():
             return
 
+        # Skip if idx is not in pre-sampled coordinates
+        if idx not in match_coords.keys():
+            warnings.warn(f"{idx} not found in {args.match_file}, skipping.")
+            return
+
         worker_start = time.time()
         patches, center_coord = get_random_patches_match(
             idx,