Support read file from http URL

docs_requirements.txt

@@ -2,6 +2,7 @@ https://download.pytorch.org/whl/cpu/torch-1.2.0%2Bcpu-cp37-cp37m-manylinux1_x86
 click
 fairseq
 future
+fvcore
 hypothesis<4.0
 mock
 numpy

pytext/__init__.py

@@ -12,7 +12,7 @@
 from pytext.data.sources.data_source import DataSource
 from pytext.task import load
 from pytext.task.new_task import NewTask
-from pytext.utils.file_io import PathManager
+from pytext.utils.file_io import PathManager, register_http_url_handler
 from pytext.workflow import _set_cuda
 
 from .builtin_task import register_builtin_tasks
@@ -21,6 +21,7 @@
 
 
 register_builtin_tasks()
+register_http_url_handler()
 
 
 Predictor = Callable[[Mapping[str, str]], Mapping[str, np.array]]

pytext/utils/file_io.py

@@ -1,57 +1,10 @@
 #!/usr/bin/env python3
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from fvcore.common.file_io import HTTPURLHandler, PathManager
 
-"""
-TODO: @stevenliu Deprecate this file after borc available in PyPI
-"""
-import os
-import shutil
-from typing import List
 
-
-try:  # noqa
-    from fvcore.common.file_io import PathManager
-
-except ImportError:
-
-    class PathManager:
-        @staticmethod
-        def open(*args, **kwargs):
-            return open(*args, **kwargs)
-
-        @staticmethod
-        def copy(*args, **kwargs) -> bool:
-            try:
-                shutil.copyfile(*args, **kwargs)
-                return True
-            except Exception as e:
-                print("Error in file copy - {}".format(str(e)))
-                return False
-
-        @staticmethod
-        def get_local_path(path: str) -> str:
-            return path
-
-        @staticmethod
-        def exists(path: str) -> bool:
-            return os.path.exists(path)
-
-        @staticmethod
-        def isfile(path: str) -> bool:
-            return os.path.isfile(path)
-
-        @staticmethod
-        def isdir(path: str) -> bool:
-            return os.path.isdir(path)
-
-        @staticmethod
-        def ls(path: str) -> List[str]:
-            return os.listdir(path)
-
-        @staticmethod
-        def mkdirs(*args, **kwargs):
-            os.makedirs(*args, exist_ok=True, **kwargs)
-
-        @staticmethod
-        def rm(*args, **kwargs):
-            os.remove(*args, **kwargs)
+def register_http_url_handler():
+    """
+    support reading file from url starting with "http://", "https://", "ftp://"
+    """
+    PathManager.register_handler(HTTPURLHandler(), allow_override=True)

requirements.txt

@@ -1,6 +1,7 @@
 click
 fairseq
 future
+fvcore
 hypothesis<4.0
 joblib
 numpy