diff --git a/docs_requirements.txt b/docs_requirements.txt index dc12fd821..1308bd23e 100644 --- a/docs_requirements.txt +++ b/docs_requirements.txt @@ -2,6 +2,7 @@ https://download.pytorch.org/whl/cpu/torch-1.2.0%2Bcpu-cp37-cp37m-manylinux1_x86 click fairseq future +fvcore hypothesis<4.0 mock numpy diff --git a/pytext/__init__.py b/pytext/__init__.py index 5e503e1fe..826390647 100644 --- a/pytext/__init__.py +++ b/pytext/__init__.py @@ -12,7 +12,7 @@ from pytext.data.sources.data_source import DataSource from pytext.task import load from pytext.task.new_task import NewTask -from pytext.utils.file_io import PathManager +from pytext.utils.file_io import PathManager, register_http_url_handler from pytext.workflow import _set_cuda from .builtin_task import register_builtin_tasks @@ -21,6 +21,7 @@ register_builtin_tasks() +register_http_url_handler() Predictor = Callable[[Mapping[str, str]], Mapping[str, np.array]] diff --git a/pytext/utils/file_io.py b/pytext/utils/file_io.py index 3daa05379..292821c47 100644 --- a/pytext/utils/file_io.py +++ b/pytext/utils/file_io.py @@ -1,57 +1,10 @@ #!/usr/bin/env python3 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +from fvcore.common.file_io import HTTPURLHandler, PathManager -""" -TODO: @stevenliu Deprecate this file after borc available in PyPI -""" -import os -import shutil -from typing import List - -try: # noqa - from fvcore.common.file_io import PathManager - -except ImportError: - - class PathManager: - @staticmethod - def open(*args, **kwargs): - return open(*args, **kwargs) - - @staticmethod - def copy(*args, **kwargs) -> bool: - try: - shutil.copyfile(*args, **kwargs) - return True - except Exception as e: - print("Error in file copy - {}".format(str(e))) - return False - - @staticmethod - def get_local_path(path: str) -> str: - return path - - @staticmethod - def exists(path: str) -> bool: - return os.path.exists(path) - - @staticmethod - def isfile(path: str) -> bool: - return os.path.isfile(path) - - @staticmethod - def isdir(path: str) -> bool: - return os.path.isdir(path) - - @staticmethod - def ls(path: str) -> List[str]: - return os.listdir(path) - - @staticmethod - def mkdirs(*args, **kwargs): - os.makedirs(*args, exist_ok=True, **kwargs) - - @staticmethod - def rm(*args, **kwargs): - os.remove(*args, **kwargs) +def register_http_url_handler(): + """ + support reading file from url starting with "http://", "https://", "ftp://" + """ + PathManager.register_handler(HTTPURLHandler(), allow_override=True) diff --git a/requirements.txt b/requirements.txt index 355896ee2..155d22a83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ click fairseq future +fvcore hypothesis<4.0 joblib numpy