Upgrade fsspec and s3fs versions

Summary: Need to update code for fsspec/filesystem_spec#1394. Reviewed By: stepanhruda Differential Revision: D50388544 fbshipit-source-id: 49944758ac7c87e828950d997ad1d6a386b125d2
pytorch · Oct 25, 2023 · a224719 · a224719
1 parent 1fe0607
commit a224719
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 14 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -4,8 +4,7 @@ importlib-metadata
 pyyaml
 docker
 filelock
-# more recent versions of fsspec causes torchx/workspace/test/dir_workspace_test#test_torchcxignore to fail
-fsspec==2023.1.0
+fsspec>=2023.9.0
 # To resolve confliciting dependencies for urllib3: https://github.com/pytorch/torchx/actions/runs/3484190429/jobs/5828784263#step:4:552
 urllib3<1.27,>=1.21.1
 tabulate
diff --git a/torchx/workspace/test/dir_workspace_test.py b/torchx/workspace/test/dir_workspace_test.py
@@ -90,23 +90,25 @@ def test_torchxignore(self) -> None:
 
         _copy_to_dir("memory://torchxignore", "memory://torchxignoredest")
 
-        files = fs.glob("torchxignoredest/*") + fs.glob("torchxignoredest/**/*")
+        files = fs.glob("torchxignoredest/**/*")
         # strip prefix
         files = [
             os.path.normpath(file.partition("torchxignoredest/")[2]) for file in files
         ]
         print(files)
-        self.assertCountEqual(
-            files,
-            {
-                ".torchxignore",
-                "dir",
-                "dir/file",
-                "dir/ignorefile",
-                "foo.sh",
-                "ignorefilesuffix",
-                "unignore",
-            },
+        self.assertListEqual(
+            sorted(files),
+            sorted(
+                [
+                    ".torchxignore",
+                    "dir",
+                    "dir/file",
+                    "dir/ignorefile",
+                    "foo.sh",
+                    "ignorefilesuffix",
+                    "unignore",
+                ]
+            ),
         )