From d43388449f9591b695cf541403c1c7044dd1fdf8 Mon Sep 17 00:00:00 2001 From: Lorenzo Mammana Date: Tue, 16 Jan 2024 17:21:08 +0100 Subject: [PATCH 1/6] build: Remove plugin dependency --- poetry.lock | 22 +++------------------- pyproject.toml | 3 +-- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index c7633bc6..02a78dbd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2134,22 +2134,6 @@ files = [ hydra-core = ">=1.1.0.dev7" optuna = ">=2.10.0,<3.0.0" -[[package]] -name = "hydra-plugins" -version = "1.0.0" -description = "Hydra plugin allowing the discovery of external configurations" -optional = false -python-versions = ">=3.9,<3.11" -files = [] -develop = false - -[package.dependencies] -poetry = "1.7.1" - -[package.source] -type = "directory" -url = "quadra_hydra_plugin" - [[package]] name = "identify" version = "2.5.33" @@ -4077,9 +4061,9 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.21.0", markers = "python_version == \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, - {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\""}, + {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, ] [[package]] @@ -4101,9 +4085,9 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.21.0", markers = "python_version == \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, - {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\""}, + {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, ] [[package]] @@ -7583,4 +7567,4 @@ test = ["pytest", "pytest_cov", "pytest_env", "pytest_lazy_fixture", "pytest_moc [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.11" -content-hash = "e45ebf4c37d3b7ad89937fa6a12837e408a6f14a298a21953b29e8617492349a" +content-hash = "d7e8605754d78a93147e6ab29a638a675c66f645c224a324fce122ea969adbb2" diff --git a/pyproject.toml b/pyproject.toml index 7480c172..58985423 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] homepage = "https://orobix.github.io/quadra" repository = "https://github.com/orobix/quadra" -packages = [{ include = "quadra" }, { include = "quadra_hydra_plugin" }] +packages = [{ include = "quadra_hydra_plugin" }, { include = "quadra" }] [build-system] requires = ["poetry-core>=1.0.0"] @@ -37,7 +37,6 @@ quadra = "quadra.main:main" [tool.poetry.dependencies] python = ">=3.9,<3.11" poetry = "1.7.1" -hydra-plugins = { path = "./quadra_hydra_plugin" } # TODO: We could support previous torch version using mutually exclusive python version but it's bad... # TODO: Right now it seems that poetry will download every kind of possible dependency from cu116 # To make it faster we could hardcode the correct version of the dependencies From b779af226ed8e496b765ced808f3f23589a2750a Mon Sep 17 00:00:00 2001 From: Lorenzo Mammana Date: Tue, 16 Jan 2024 17:57:09 +0100 Subject: [PATCH 2/6] wip: Testing stuff --- pyproject.toml | 2 +- .../hydra_plugins/quadra_searchpath_plugin.py | 30 ------------------- quadra_hydra_plugin/pyproject.toml | 19 ------------ 3 files changed, 1 insertion(+), 50 deletions(-) delete mode 100644 quadra_hydra_plugin/hydra_plugins/quadra_searchpath_plugin.py delete mode 100644 quadra_hydra_plugin/pyproject.toml diff --git a/pyproject.toml b/pyproject.toml index 58985423..e06b4085 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] homepage = "https://orobix.github.io/quadra" repository = "https://github.com/orobix/quadra" -packages = [{ include = "quadra_hydra_plugin" }, { include = "quadra" }] +packages = [{ include = "hydra_plugins" }, { include = "quadra" }] [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/quadra_hydra_plugin/hydra_plugins/quadra_searchpath_plugin.py b/quadra_hydra_plugin/hydra_plugins/quadra_searchpath_plugin.py deleted file mode 100644 index 57c5ac85..00000000 --- a/quadra_hydra_plugin/hydra_plugins/quadra_searchpath_plugin.py +++ /dev/null @@ -1,30 +0,0 @@ -import os - -import dotenv -from hydra.core.config_search_path import ConfigSearchPath -from hydra.plugins.search_path_plugin import SearchPathPlugin - - -class QuadraSearchPathPlugin(SearchPathPlugin): - """Generic Search Path Plugin class.""" - - def __init__(self): - try: - os.getcwd() - except FileNotFoundError: - # This may happen when running tests - return - - if os.path.exists(os.path.join(os.getcwd(), ".env")): - dotenv.load_dotenv(dotenv_path=os.path.join(os.getcwd(), ".env"), override=True) - - def manipulate_search_path(self, search_path: ConfigSearchPath) -> None: - """Plugin used to add custom config to searchpath to be discovered by quadra.""" - # This can be global or taken from the .env - quadra_search_path = os.environ.get("QUADRA_SEARCH_PATH", None) - - # Path should be specified as a list of hydra path separated by ";" - # E.g pkg://package1.configs;file:///path/to/configs - if quadra_search_path is not None: - for i, path in enumerate(quadra_search_path.split(";")): - search_path.append(provider=f"quadra-searchpath-plugin-{i}", path=path) diff --git a/quadra_hydra_plugin/pyproject.toml b/quadra_hydra_plugin/pyproject.toml deleted file mode 100644 index 65769a92..00000000 --- a/quadra_hydra_plugin/pyproject.toml +++ /dev/null @@ -1,19 +0,0 @@ -[tool.poetry] -name = "hydra_plugins" -version = "1.0.0" -description = "Hydra plugin allowing the discovery of external configurations" -authors = [ - "Federico Belotti ", - "Silvia Bianchetti ", - "Refik Can Malli ", - "Lorenzo Mammana ", - "Alessandro Polidori ", -] - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" - -[tool.poetry.dependencies] -python = ">=3.9,<3.11" -poetry = "1.7.1" From f2d46668a3673fac45ff0cbb9b47ebeae1abbae8 Mon Sep 17 00:00:00 2001 From: Lorenzo Mammana Date: Tue, 16 Jan 2024 18:57:39 +0100 Subject: [PATCH 3/6] build: Add missing hydra_plugins folder --- hydra_plugins/quadra_searchpath_plugin.py | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 hydra_plugins/quadra_searchpath_plugin.py diff --git a/hydra_plugins/quadra_searchpath_plugin.py b/hydra_plugins/quadra_searchpath_plugin.py new file mode 100644 index 00000000..57c5ac85 --- /dev/null +++ b/hydra_plugins/quadra_searchpath_plugin.py @@ -0,0 +1,30 @@ +import os + +import dotenv +from hydra.core.config_search_path import ConfigSearchPath +from hydra.plugins.search_path_plugin import SearchPathPlugin + + +class QuadraSearchPathPlugin(SearchPathPlugin): + """Generic Search Path Plugin class.""" + + def __init__(self): + try: + os.getcwd() + except FileNotFoundError: + # This may happen when running tests + return + + if os.path.exists(os.path.join(os.getcwd(), ".env")): + dotenv.load_dotenv(dotenv_path=os.path.join(os.getcwd(), ".env"), override=True) + + def manipulate_search_path(self, search_path: ConfigSearchPath) -> None: + """Plugin used to add custom config to searchpath to be discovered by quadra.""" + # This can be global or taken from the .env + quadra_search_path = os.environ.get("QUADRA_SEARCH_PATH", None) + + # Path should be specified as a list of hydra path separated by ";" + # E.g pkg://package1.configs;file:///path/to/configs + if quadra_search_path is not None: + for i, path in enumerate(quadra_search_path.split(";")): + search_path.append(provider=f"quadra-searchpath-plugin-{i}", path=path) From f8b4a394734e511b7b05b9712f39b1696a33b4db Mon Sep 17 00:00:00 2001 From: Lorenzo Mammana Date: Wed, 17 Jan 2024 09:50:48 +0100 Subject: [PATCH 4/6] refactor: Reorder packages --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e06b4085..47749874 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] homepage = "https://orobix.github.io/quadra" repository = "https://github.com/orobix/quadra" -packages = [{ include = "hydra_plugins" }, { include = "quadra" }] +packages = [{ include = "quadra" }, { include = "hydra_plugins" }] [build-system] requires = ["poetry-core>=1.0.0"] From f57b8a60787d777d80e1dea049ea3e91cbd7051a Mon Sep 17 00:00:00 2001 From: Lorenzo Mammana Date: Wed, 17 Jan 2024 10:47:41 +0100 Subject: [PATCH 5/6] build: Include hydra plugin as dev dependecy --- poetry.lock | 17 +++++++++++++++-- pyproject.toml | 8 +++++++- .../hydra_plugins}/quadra_searchpath_plugin.py | 0 quadra_hydra_plugin/pyproject.toml | 15 +++++++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) rename {hydra_plugins => quadra_hydra_plugin/hydra_plugins}/quadra_searchpath_plugin.py (100%) create mode 100644 quadra_hydra_plugin/pyproject.toml diff --git a/poetry.lock b/poetry.lock index 02a78dbd..a8f80c97 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2134,6 +2134,19 @@ files = [ hydra-core = ">=1.1.0.dev7" optuna = ">=2.10.0,<3.0.0" +[[package]] +name = "hydra-plugins" +version = "1.0.0" +description = "Hydra plugin allowing the discovery of external configurations" +optional = true +python-versions = "*" +files = [] +develop = false + +[package.source] +type = "directory" +url = "quadra_hydra_plugin" + [[package]] name = "identify" version = "2.5.33" @@ -7559,7 +7572,7 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] [extras] -dev = ["black", "bump2version", "cairosvg", "interrogate", "isort", "mike", "mkdocs", "mkdocs_autorefs", "mkdocs_gen_files", "mkdocs_literate_nav", "mkdocs_material", "mkdocs_material_extensions", "mkdocs_section_index", "mkdocstrings", "mkdocstrings_python", "mypy", "pandas_stubs", "poetry-dynamic-versioning", "pre_commit", "pylint", "pytest", "pytest_cov", "pytest_env", "pytest_lazy_fixture", "pytest_mock", "ruff", "twine", "types_pyyaml"] +dev = ["black", "bump2version", "cairosvg", "hydra-plugins", "interrogate", "isort", "mike", "mkdocs", "mkdocs_autorefs", "mkdocs_gen_files", "mkdocs_literate_nav", "mkdocs_material", "mkdocs_material_extensions", "mkdocs_section_index", "mkdocstrings", "mkdocstrings_python", "mypy", "pandas_stubs", "poetry-dynamic-versioning", "pre_commit", "pylint", "pytest", "pytest_cov", "pytest_env", "pytest_lazy_fixture", "pytest_mock", "ruff", "twine", "types_pyyaml"] docs = ["cairosvg", "mike", "mkdocs", "mkdocs_autorefs", "mkdocs_gen_files", "mkdocs_literate_nav", "mkdocs_material", "mkdocs_material_extensions", "mkdocs_section_index", "mkdocstrings", "mkdocstrings_python"] onnx = ["onnx", "onnxruntime_gpu", "onnxsim"] test = ["pytest", "pytest_cov", "pytest_env", "pytest_lazy_fixture", "pytest_mock"] @@ -7567,4 +7580,4 @@ test = ["pytest", "pytest_cov", "pytest_env", "pytest_lazy_fixture", "pytest_moc [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.11" -content-hash = "d7e8605754d78a93147e6ab29a638a675c66f645c224a324fce122ea969adbb2" +content-hash = "89770057e6656cc797382a515b29729a19969e03542a16ab0a5ae1219214516c" diff --git a/pyproject.toml b/pyproject.toml index 47749874..039096ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,10 @@ classifiers = [ ] homepage = "https://orobix.github.io/quadra" repository = "https://github.com/orobix/quadra" -packages = [{ include = "quadra" }, { include = "hydra_plugins" }] +packages = [ + { include = "quadra" }, + { include = "hydra_plugins", from = "quadra_hydra_plugin" }, +] [build-system] requires = ["poetry-core>=1.0.0"] @@ -37,6 +40,8 @@ quadra = "quadra.main:main" [tool.poetry.dependencies] python = ">=3.9,<3.11" poetry = "1.7.1" +# This will make hydra-plugins available also when running with poetry install +hydra-plugins = { path = "quadra_hydra_plugin", optional = true } # TODO: We could support previous torch version using mutually exclusive python version but it's bad... # TODO: Right now it seems that poetry will download every kind of possible dependency from cu116 # To make it faster we could hardcode the correct version of the dependencies @@ -152,6 +157,7 @@ dev = [ "mike", "cairosvg", "poetry-dynamic-versioning", + "hydra-plugins", ] test = [ diff --git a/hydra_plugins/quadra_searchpath_plugin.py b/quadra_hydra_plugin/hydra_plugins/quadra_searchpath_plugin.py similarity index 100% rename from hydra_plugins/quadra_searchpath_plugin.py rename to quadra_hydra_plugin/hydra_plugins/quadra_searchpath_plugin.py diff --git a/quadra_hydra_plugin/pyproject.toml b/quadra_hydra_plugin/pyproject.toml new file mode 100644 index 00000000..b435e557 --- /dev/null +++ b/quadra_hydra_plugin/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "hydra_plugins" +version = "1.0.0" +description = "Hydra plugin allowing the discovery of external configurations" +authors = [ + "Federico Belotti ", + "Silvia Bianchetti ", + "Refik Can Malli ", + "Lorenzo Mammana ", + "Alessandro Polidori ", +] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" From 941421e42da0331d7c30b642b2003560b4e48d99 Mon Sep 17 00:00:00 2001 From: Lorenzo Mammana Date: Wed, 17 Jan 2024 11:34:12 +0100 Subject: [PATCH 6/6] build: Upgrade version to 1.5.2 --- CHANGELOG.md | 6 ++++++ pyproject.toml | 2 +- quadra/__init__.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4032b27f..398421ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ # Changelog All notable changes to this project will be documented in this file. +### [1.5.2] + +#### Fixed + +- Fix hydra plugin not working properly when the library is installed from external sources. + ### [1.5.1] #### Fixed diff --git a/pyproject.toml b/pyproject.toml index 039096ae..5c3d2bda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "quadra" -version = "1.5.1" +version = "1.5.2" description = "Deep Learning experiment orchestration library" authors = [ "Federico Belotti ", diff --git a/quadra/__init__.py b/quadra/__init__.py index ce214805..e7e614b1 100644 --- a/quadra/__init__.py +++ b/quadra/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.5.1" +__version__ = "1.5.2" def get_version():