From a446b3499e7b1d090d714c1a1966d99599578fa3 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 5 Aug 2024 16:02:13 +0200 Subject: [PATCH 1/5] Remove hub references --- ...ecify-pip-dependencies-and-apt-packages.md | 1 - src/zenml/__init__.py | 4 - src/zenml/_hub/__init__.py | 14 - src/zenml/_hub/client.py | 289 ----- src/zenml/_hub/constants.py | 21 - src/zenml/_hub/utils.py | 79 -- src/zenml/analytics/enums.py | 6 - src/zenml/cli/__init__.py | 61 - src/zenml/cli/hub.py | 1116 ----------------- src/zenml/config/docker_settings.py | 9 +- src/zenml/constants.py | 2 - src/zenml/enums.py | 1 - src/zenml/logger.py | 2 +- src/zenml/models/__init__.py | 12 - src/zenml/models/v2/core/user.py | 21 - src/zenml/models/v2/misc/hub_plugin_models.py | 79 -- .../utils/pipeline_docker_image_builder.py | 95 +- .../zen_server/routers/users_endpoints.py | 7 - .../909550c7c4da_remove_user_hub_token.py | 36 + src/zenml/zen_stores/schemas/user_schemas.py | 1 - tests/unit/_hub/test_client.py | 60 - tests/unit/_hub/test_utils.py | 79 -- 22 files changed, 41 insertions(+), 1954 deletions(-) delete mode 100644 src/zenml/_hub/__init__.py delete mode 100644 src/zenml/_hub/client.py delete mode 100644 src/zenml/_hub/constants.py delete mode 100644 src/zenml/_hub/utils.py delete mode 100644 src/zenml/cli/hub.py delete mode 100644 src/zenml/models/v2/misc/hub_plugin_models.py create mode 100644 src/zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py delete mode 100644 tests/unit/_hub/test_client.py delete mode 100644 tests/unit/_hub/test_utils.py diff --git a/docs/book/how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md b/docs/book/how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md index 96cb71a36aa..46c91f00af1 100644 --- a/docs/book/how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md +++ b/docs/book/how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md @@ -103,7 +103,6 @@ You can combine these methods but do make sure that your list of requirements do Depending on the options specified in your Docker settings, ZenML installs the requirements in the following order (each step optional): * The packages installed in your local python environment -* The packages specified via the `required_hub_plugins` attribute * The packages required by the stack unless this is disabled by setting `install_stack_requirements=False`. * The packages specified via the `required_integrations` * The packages specified via the `requirements` attribute diff --git a/src/zenml/__init__.py b/src/zenml/__init__.py index 36a2d90d7c2..0f8e22aeb8f 100644 --- a/src/zenml/__init__.py +++ b/src/zenml/__init__.py @@ -27,10 +27,6 @@ init_logging() -# The following code is needed for `zenml.hub` subpackages to be found -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) # Need to import zenml.models before zenml.config to avoid circular imports from zenml.models import * # noqa: F401 diff --git a/src/zenml/_hub/__init__.py b/src/zenml/_hub/__init__.py deleted file mode 100644 index c1b484f8826..00000000000 --- a/src/zenml/_hub/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) ZenML GmbH 2023. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing -# permissions and limitations under the License. -"""ZenML Hub internal code.""" diff --git a/src/zenml/_hub/client.py b/src/zenml/_hub/client.py deleted file mode 100644 index c9c861e1ab9..00000000000 --- a/src/zenml/_hub/client.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright (c) ZenML GmbH 2023. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing -# permissions and limitations under the License. -"""Client for the ZenML Hub.""" - -import os -from json import JSONDecodeError -from typing import Any, Dict, List, Optional - -import requests - -from zenml._hub.constants import ( - ZENML_HUB_ADMIN_USERNAME, - ZENML_HUB_CLIENT_TIMEOUT, - ZENML_HUB_CLIENT_VERIFY, - ZENML_HUB_DEFAULT_URL, -) -from zenml.analytics import source_context -from zenml.client import Client -from zenml.constants import ENV_ZENML_HUB_URL, IS_DEBUG_ENV -from zenml.logger import get_logger -from zenml.models import ( - HubPluginRequestModel, - HubPluginResponseModel, - HubUserResponseModel, -) - -logger = get_logger(__name__) - - -class HubAPIError(Exception): - """Exception raised when the Hub returns an error or unexpected response.""" - - -class HubClient: - """Client for the ZenML Hub.""" - - def __init__(self, url: Optional[str] = None) -> None: - """Initialize the client. - - Args: - url: The URL of the ZenML Hub. - """ - self.url = url or self.get_default_url() - self.auth_token = Client().active_user.hub_token - - @staticmethod - def get_default_url() -> str: - """Get the default URL of the ZenML Hub. - - Returns: - The default URL of the ZenML Hub. - """ - return os.getenv(ENV_ZENML_HUB_URL, default=ZENML_HUB_DEFAULT_URL) - - def list_plugins(self, **params: Any) -> List[HubPluginResponseModel]: - """List all plugins in the hub. - - Args: - **params: The query parameters to send in the request. - - Returns: - The list of plugin response models. - """ - response = self._request("GET", "/plugins", params=params) - if not isinstance(response, list): - return [] - return [ - HubPluginResponseModel.model_validate(plugin) - for plugin in response - ] - - def get_plugin( - self, - name: str, - version: Optional[str] = None, - author: Optional[str] = None, - ) -> Optional[HubPluginResponseModel]: - """Get a specific plugin from the hub. - - Args: - name: The name of the plugin. - version: The version of the plugin. If not specified, the latest - version will be returned. - author: Username of the author of the plugin. - - Returns: - The plugin response model or None if the plugin does not exist. - """ - route = "/plugins" - if not author: - author = ZENML_HUB_ADMIN_USERNAME - options = [ - f"name={name}", - f"username={author}", - ] - if version: - options.append(f"version={version}") - if options: - route += "?" + "&".join(options) - try: - response = self._request("GET", route) - except HubAPIError: - return None - if not isinstance(response, list) or len(response) == 0: - return None - return HubPluginResponseModel.model_validate(response[0]) - - def create_plugin( - self, plugin_request: HubPluginRequestModel - ) -> HubPluginResponseModel: - """Create a plugin in the hub. - - Args: - plugin_request: The plugin request model. - - Returns: - The plugin response model. - """ - route = "/plugins" - response = self._request( - "POST", route, data=plugin_request.model_dump_json() - ) - return HubPluginResponseModel.model_validate(response) - - # TODO: Potentially reenable this later if hub adds logs streaming endpoint - # def stream_plugin_build_logs( - # self, plugin_name: str, plugin_version: str - # ) -> bool: - # """Stream the build logs of a plugin. - - # Args: - # plugin_name: The name of the plugin. - # plugin_version: The version of the plugin. If not specified, the - # latest version will be used. - - # Returns: - # Whether any logs were found. - - # Raises: - # HubAPIError: If the build failed. - # """ - # route = f"plugins/{plugin_name}/versions/{plugin_version}/logs" - # logs_url = os.path.join(self.url, route) - - # found_logs = False - # with requests.get(logs_url, stream=True) as response: - # for line in response.iter_lines( - # chunk_size=None, decode_unicode=True - # ): - # found_logs = True - # if line.startswith("Build failed"): - # raise HubAPIError(line) - # else: - # logger.info(line) - # return found_logs - - def login(self, username: str, password: str) -> None: - """Login to the ZenML Hub. - - Args: - username: The username of the user in the ZenML Hub. - password: The password of the user in the ZenML Hub. - - Raises: - HubAPIError: If the login failed. - """ - route = "/auth/jwt/login" - response = self._request( - method="POST", - route=route, - data={"username": username, "password": password}, - content_type="application/x-www-form-urlencoded", - ) - if isinstance(response, dict): - auth_token = response.get("access_token") - if auth_token: - self.set_auth_token(str(auth_token)) - return - raise HubAPIError(f"Unexpected response: {response}") - - def set_auth_token(self, auth_token: Optional[str]) -> None: - """Set the auth token. - - Args: - auth_token: The auth token to set. - """ - client = Client() - client.update_user( - name_id_or_prefix=client.active_user.id, - updated_hub_token=auth_token, - ) - self.auth_token = auth_token - - def get_github_login_url(self) -> str: - """Get the GitHub login URL. - - Returns: - The GitHub login URL. - - Raises: - HubAPIError: If the request failed. - """ - route = "/auth/github/authorize" - response = self._request("GET", route) - if isinstance(response, dict): - auth_url = response.get("authorization_url") - if auth_url: - return str(auth_url) - raise HubAPIError(f"Unexpected response: {str(response)}") - - def get_me(self) -> Optional[HubUserResponseModel]: - """Get the current user. - - Returns: - The user response model or None if the user does not exist. - """ - try: - response = self._request("GET", "/users/me") - return HubUserResponseModel.model_validate(response) - except HubAPIError: - return None - - def _request( - self, - method: str, - route: str, - data: Optional[Any] = None, - params: Optional[Dict[str, Any]] = None, - content_type: str = "application/json", - ) -> Any: - """Helper function to make a request to the hub. - - Args: - method: The HTTP method to use. - route: The route to send the request to, e.g., "/plugins". - data: The data to send in the request. - params: The query parameters to send in the request. - content_type: The content type of the request. - - Returns: - The response JSON. - - Raises: - HubAPIError: If the request failed. - """ - session = requests.Session() - - # Define headers - headers = { - "Accept": "application/json", - "Content-Type": content_type, - "Debug-Context": str(IS_DEBUG_ENV), - "Source-Context": str(source_context.get().value), - } - if self.auth_token: - headers["Authorization"] = f"Bearer {self.auth_token}" - - # Make the request - route = route.lstrip("/") - endpoint_url = os.path.join(self.url, route) - response = session.request( - method=method, - url=endpoint_url, - data=data, - headers=headers, - params=params, - verify=ZENML_HUB_CLIENT_VERIFY, - timeout=ZENML_HUB_CLIENT_TIMEOUT, - ) - - # Parse and return the response - if 200 <= response.status_code < 300: - return response.json() - try: - error_msg = response.json().get("detail", response.text) - except JSONDecodeError: - error_msg = response.text - raise HubAPIError(f"Request to ZenML Hub failed: {error_msg}") diff --git a/src/zenml/_hub/constants.py b/src/zenml/_hub/constants.py deleted file mode 100644 index 702a2a3cbc7..00000000000 --- a/src/zenml/_hub/constants.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) ZenML GmbH 2023. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing -# permissions and limitations under the License. -"""Constants for the ZenML hub.""" - -ZENML_HUB_DEFAULT_URL = "https://hubapi.zenml.io/" -ZENML_HUB_ADMIN_USERNAME = "ZenML" -ZENML_HUB_CLIENT_VERIFY = True -ZENML_HUB_CLIENT_TIMEOUT = 10 -ZENML_HUB_INTERNAL_TAG_PREFIX = "zenml-" -ZENML_HUB_VERIFIED_TAG = "zenml-badge-verified" diff --git a/src/zenml/_hub/utils.py b/src/zenml/_hub/utils.py deleted file mode 100644 index 517def2a180..00000000000 --- a/src/zenml/_hub/utils.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) ZenML GmbH 2023. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing -# permissions and limitations under the License. -"""Utility functions for the ZenML Hub.""" - -from typing import Optional, Tuple - -from zenml._hub.constants import ZENML_HUB_ADMIN_USERNAME - - -def parse_plugin_name( - plugin_name: str, author_separator: str = "/", version_separator: str = ":" -) -> Tuple[Optional[str], str, str]: - """Helper function to parse a plugin name. - - Args: - plugin_name: The user-provided plugin name. - author_separator: The separator between the author username and the - plugin name. - version_separator: The separator between the plugin name and the - plugin version. - - Returns: - - The author username or None if no author was specified. - - The plugin name. - - The plugin version or "latest" if no version was specified. - - Raises: - ValueError: If the plugin name is invalid. - """ - invalid_format_err_msg = ( - f"Invalid plugin name '{plugin_name}'. Expected format: " - f"`({author_separator})({version_separator})`." - ) - - parts = plugin_name.split(version_separator) - if len(parts) > 2: - raise ValueError(invalid_format_err_msg) - name, version = parts[0], "latest" if len(parts) == 1 else parts[1] - - parts = name.split(author_separator) - if len(parts) > 2: - raise ValueError(invalid_format_err_msg) - name, author = parts[-1], None if len(parts) == 1 else parts[0] - - if not name: - raise ValueError(invalid_format_err_msg) - - return author, name, version - - -def plugin_display_name( - name: str, version: Optional[str], author: Optional[str] -) -> str: - """Helper function to get the display name of a plugin. - - Args: - name: Name of the plugin. - version: Version of the plugin. - author: Username of the plugin author. - - Returns: - Display name of the plugin. - """ - author_prefix = "" - if author and author != ZENML_HUB_ADMIN_USERNAME: - author_prefix = f"{author}/" - version_suffix = f":{version}" if version else ":latest" - return f"{author_prefix}{name}{version_suffix}" diff --git a/src/zenml/analytics/enums.py b/src/zenml/analytics/enums.py index c617b32e145..d599bbf527f 100644 --- a/src/zenml/analytics/enums.py +++ b/src/zenml/analytics/enums.py @@ -99,11 +99,5 @@ class AnalyticsEvent(str, Enum): ZENML_SERVER_DEPLOYED = "ZenML server deployed" ZENML_SERVER_DESTROYED = "ZenML server destroyed" - # ZenML Hub events - ZENML_HUB_PLUGIN_INSTALL = "ZenML Hub plugin installed" - ZENML_HUB_PLUGIN_UNINSTALL = "ZenML Hub plugin uninstalled" - ZENML_HUB_PLUGIN_CLONE = "ZenML Hub plugin pulled" - ZENML_HUB_PLUGIN_SUBMIT = "ZenML Hub plugin pushed" - # Server Settings SERVER_SETTINGS_UPDATED = "Server Settings Updated" diff --git a/src/zenml/cli/__init__.py b/src/zenml/cli/__init__.py index 29b3c7d87cc..6860c75f1c1 100644 --- a/src/zenml/cli/__init__.py +++ b/src/zenml/cli/__init__.py @@ -2541,66 +2541,6 @@ def my_pipeline(...): For full documentation on this functionality, please refer to [the dedicated documentation on stack component deploy](https://docs.zenml.io/how-to/stack-deployment/deploy-a-stack-component). - -Interacting with the ZenML Hub ------------------------------- - -The ZenML Hub is a central location for discovering and sharing third-party -ZenML code, such as custom integrations, components, steps, pipelines, -materializers, and more. -You can browse the ZenML Hub at [https://hub.zenml.io](https://hub.zenml.io). - -The ZenML CLI provides various commands to interact with the ZenML Hub: - -- Listing all plugins available on the Hub: -```bash -zenml hub list -``` - -- Installing a Hub plugin: -```bash -zenml hub install -``` -Installed plugins can be imported via `from zenml.hub. import ...`. - - -- Uninstalling a Hub plugin: -```bash -zenml hub uninstall -``` - -- Cloning the source code of a Hub plugin (without installing it): -```bash -zenml hub clone -``` -This is useful, e.g., for extending an existing plugin or for getting the -examples of a plugin. - -- Submitting/contributing a plugin to the Hub (requires login, see below): -```bash -zenml hub submit -``` -If you are unsure about which arguments you need to set, you can run the -command in interactive mode: -```bash -zenml hub submit --interactive -``` -This will ask for and validate inputs one at a time. - -- Logging in to the Hub: -```bash -zenml hub login -``` - -- Logging out of the Hub: -```bash -zenml hub logout -``` - -- Viewing the build logs of a plugin you submitted to the Hub: -```bash -zenml hub logs -``` """ from zenml.cli.version import * # noqa @@ -2612,7 +2552,6 @@ def my_pipeline(...): from zenml.cli.config import * # noqa from zenml.cli.downgrade import * # noqa from zenml.cli.feature import * # noqa -from zenml.cli.hub import * # noqa from zenml.cli.integration import * # noqa from zenml.cli.model import * # noqa from zenml.cli.model_registry import * # noqa diff --git a/src/zenml/cli/hub.py b/src/zenml/cli/hub.py deleted file mode 100644 index 3f9d84f2215..00000000000 --- a/src/zenml/cli/hub.py +++ /dev/null @@ -1,1116 +0,0 @@ -# Copyright (c) ZenML GmbH 2023. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing -# permissions and limitations under the License. -"""CLI functionality to interact with the ZenML Hub.""" - -import os -import shutil -import subprocess -import sys -import tempfile -from importlib.util import find_spec -from typing import Any, Dict, List, Optional, Tuple - -import click - -from zenml._hub.client import HubAPIError, HubClient -from zenml._hub.constants import ( - ZENML_HUB_ADMIN_USERNAME, - ZENML_HUB_INTERNAL_TAG_PREFIX, - ZENML_HUB_VERIFIED_TAG, -) -from zenml._hub.utils import parse_plugin_name, plugin_display_name -from zenml.analytics.enums import AnalyticsEvent -from zenml.analytics.utils import track_handler -from zenml.cli.cli import TagGroup, cli -from zenml.cli.utils import declare, error, print_table, warning -from zenml.enums import CliCategories -from zenml.logger import get_logger -from zenml.models import ( - HubPluginRequestModel, - HubPluginResponseModel, - PluginStatus, -) - -logger = get_logger(__name__) - - -@cli.group(cls=TagGroup, tag=CliCategories.HUB) -def hub() -> None: - """Interact with the ZenML Hub.""" - - -@hub.command("list") -@click.option( - "--all", - "-a", - is_flag=True, - help="List all plugins, including those that are not available.", -) -@click.option( - "--mine", - "-m", - is_flag=True, - help="List only plugins that you own.", -) -@click.option( - "--installed", - "-i", - is_flag=True, - help="List only plugins that are installed.", -) -def list_plugins( - all: bool = False, mine: bool = False, installed: bool = False -) -> None: - """List all plugins available on the ZenML Hub. - - Args: - all: Whether to list all plugins, including ones that are not available. - mine: Whether to list only plugins that you own. - installed: Whether to list only plugins that are installed. - """ - client = HubClient() - if mine and not client.auth_token: - error( - "You must be logged in to list your own plugins via --mine. " - "Please run `zenml hub login` to login." - ) - list_params: Dict[str, Any] = {"mine": mine} - if not all: - list_params["status"] = PluginStatus.AVAILABLE - plugins = client.list_plugins(**list_params) - if not plugins: - declare("No plugins found.") - if installed: - plugins = [ - plugin - for plugin in plugins - if _is_plugin_installed( - author=plugin.author, plugin_name=plugin.name - ) - ] - plugins_table = _format_plugins_table(plugins) - print_table(plugins_table) - - -def _format_plugins_table( - plugins: List[HubPluginResponseModel], -) -> List[Dict[str, str]]: - """Helper function to format a list of plugins into a table. - - Args: - plugins: The list of plugins. - - Returns: - The formatted table. - """ - plugins_table = [] - for plugin in sorted(plugins, key=_sort_plugin_key_fn): - if _is_plugin_installed(author=plugin.author, plugin_name=plugin.name): - installed_icon = ":white_check_mark:" - else: - installed_icon = ":x:" - if plugin.status == PluginStatus.AVAILABLE: - status_icon = ":white_check_mark:" - elif plugin.status == PluginStatus.PENDING: - status_icon = ":hourglass:" - else: - status_icon = ":x:" - - display_name = plugin_display_name( - name=plugin.name, - version=plugin.version, - author=plugin.author, - ) - display_data: Dict[str, str] = { - "PLUGIN": display_name, - "STATUS": status_icon, - "INSTALLED": installed_icon, - "MODULE": _get_plugin_module( - author=plugin.author, plugin_name=plugin.name - ), - "PACKAGE NAME": plugin.package_name or "", - "REPOSITORY URL": plugin.repository_url, - } - plugins_table.append(display_data) - return plugins_table - - -def _sort_plugin_key_fn( - plugin: HubPluginResponseModel, -) -> Tuple[str, str, str]: - """Helper function to sort plugins by name, version and author. - - Args: - plugin: The plugin to sort. - - Returns: - A tuple of the plugin's author, name and version. - """ - if plugin.author == ZENML_HUB_ADMIN_USERNAME: - return "0", plugin.name, plugin.version # Sort admin plugins first - return plugin.author, plugin.name, plugin.version - - -@hub.command("install") -@click.argument("plugin_name", type=str, required=True) -@click.option( - "--upgrade", - "-u", - is_flag=True, - help="Upgrade the plugin if it is already installed.", -) -@click.option( - "--no-deps", - "--no-dependencies", - is_flag=True, - help="Do not install dependencies of the plugin.", -) -@click.option( - "--yes", - "-y", - is_flag=True, - help="Do not ask for confirmation before installing.", -) -def install_plugin( - plugin_name: str, - upgrade: bool = False, - no_deps: bool = False, - yes: bool = False, -) -> None: - """Install a plugin from the ZenML Hub. - - Args: - plugin_name: Name of the plugin. - upgrade: Whether to upgrade the plugin if it is already installed. - no_deps: If set, dependencies of the plugin will not be installed. - yes: If set, no confirmation will be asked for before installing. - """ - with track_handler( - event=AnalyticsEvent.ZENML_HUB_PLUGIN_INSTALL, - ) as analytics_handler: - client = HubClient() - analytics_handler.metadata["hub_url"] = client.url - author, plugin_name, plugin_version = parse_plugin_name(plugin_name) - analytics_handler.metadata["plugin_name"] = plugin_name - analytics_handler.metadata["plugin_author"] = author - display_name = plugin_display_name(plugin_name, plugin_version, author) - - # Get plugin from hub - plugin = client.get_plugin( - name=plugin_name, - version=plugin_version, - author=author, - ) - if not plugin: - error(f"Could not find plugin '{display_name}' on the hub.") - analytics_handler.metadata["plugin_version"] = plugin.version - - # Check if plugin can be installed - index_url = plugin.index_url - package_name = plugin.package_name - if not index_url or not package_name: - error( - f"Plugin '{display_name}' is not available for installation." - ) - - # Check if plugin is already installed - if ( - _is_plugin_installed(author=plugin.author, plugin_name=plugin.name) - and not upgrade - ): - declare(f"Plugin '{plugin_name}' is already installed.") - return - - # Show a warning if the plugin is not official or verified - _is_zenml_plugin = plugin.author == ZENML_HUB_ADMIN_USERNAME - _is_verified = plugin.tags and ZENML_HUB_VERIFIED_TAG in plugin.tags - if not _is_zenml_plugin and not _is_verified: - warning( - f"Plugin '{display_name}' was not verified by ZenML and may " - "contain arbitrary code. Please check the source code before " - "installing to make sure it does what you expect." - ) - - # Install plugin requirements - if plugin.requirements: - requirements_str = " ".join(f"'{r}'" for r in plugin.requirements) - install_requirements = False - if not no_deps and not yes: - install_requirements = click.confirm( - f"Plugin '{display_name}' requires the following " - f"packages to be installed: {requirements_str}. " - f"Do you want to install them now?" - ) - if yes or install_requirements: - declare( - f"Installing requirements for plugin '{display_name}': " - f"{requirements_str}..." - ) - requirements_install_call = [ - sys.executable, - "-m", - "pip", - "install", - *list(plugin.requirements), - "--upgrade", - ] - subprocess.check_call(requirements_install_call) - declare( - f"Successfully installed requirements for plugin " - f"'{display_name}'." - ) - else: - warning( - f"Requirements for plugin '{display_name}' were not " - "installed. This might lead to errors in the future if the " - "requirements are not installed manually." - ) - - # pip install the wheel - declare( - f"Installing plugin '{display_name}' from " - f"{index_url}{package_name}..." - ) - plugin_install_call = [ - sys.executable, - "-m", - "pip", - "install", - "--index-url", - index_url, - package_name, - "--no-deps", # we already installed the requirements above - "--upgrade", # we already checked if the plugin is installed above - ] - subprocess.check_call(plugin_install_call) - declare(f"Successfully installed plugin '{display_name}'.") - - -@hub.command("uninstall") -@click.argument("plugin_name", type=str, required=True) -def uninstall_plugin(plugin_name: str) -> None: - """Uninstall a ZenML Hub plugin. - - Args: - plugin_name: Name of the plugin. - """ - with track_handler( - event=AnalyticsEvent.ZENML_HUB_PLUGIN_UNINSTALL, - ) as analytics_handler: - client = HubClient() - analytics_handler.metadata["hub_url"] = client.url - author, plugin_name, plugin_version = parse_plugin_name(plugin_name) - analytics_handler.metadata["plugin_name"] = plugin_name - analytics_handler.metadata["plugin_author"] = author - display_name = plugin_display_name(plugin_name, plugin_version, author) - - # Get plugin from hub - plugin = client.get_plugin( - name=plugin_name, - version=plugin_version, - author=author, - ) - if not plugin: - error(f"Could not find plugin '{display_name}' on the hub.") - analytics_handler.metadata["plugin_version"] = plugin.version - - # Check if plugin can be uninstalled - package_name = plugin.package_name - if not package_name: - error( - f"Plugin '{display_name}' is not available for uninstallation." - ) - - # pip uninstall the wheel - declare(f"Uninstalling plugin '{display_name}'...") - subprocess.check_call( - [sys.executable, "-m", "pip", "uninstall", package_name, "-y"] - ) - declare(f"Successfully uninstalled plugin '{display_name}'.") - - -@hub.command("clone") -@click.argument("plugin_name", type=str, required=True) -@click.option( - "--output_dir", - "-o", - type=str, - help="Output directory to clone the plugin to.", -) -def clone_plugin( - plugin_name: str, - output_dir: Optional[str] = None, -) -> None: - """Clone the source code of a ZenML Hub plugin. - - Args: - plugin_name: Name of the plugin. - output_dir: Output directory to clone the plugin to. If not specified, - the plugin will be cloned to a directory with the same name as the - plugin in the current working directory. - """ - from zenml.utils.git_utils import clone_git_repository - - with track_handler( - event=AnalyticsEvent.ZENML_HUB_PLUGIN_CLONE, - ) as analytics_handler: - client = HubClient() - analytics_handler.metadata["hub_url"] = client.url - author, plugin_name, plugin_version = parse_plugin_name(plugin_name) - analytics_handler.metadata["plugin_name"] = plugin_name - analytics_handler.metadata["plugin_author"] = author - display_name = plugin_display_name(plugin_name, plugin_version, author) - - # Get plugin from hub - plugin = client.get_plugin( - name=plugin_name, - version=plugin_version, - author=author, - ) - if not plugin: - error(f"Could not find plugin '{display_name}' on the hub.") - analytics_handler.metadata["plugin_version"] = plugin.version - - # Clone the source repo to a temp dir, then move the plugin subdir out - repo_url = plugin.repository_url - subdir = plugin.repository_subdirectory - commit = plugin.repository_commit - if output_dir is None: - output_dir = os.path.join(os.getcwd(), plugin_name) - declare(f"Cloning plugin '{display_name}' to {output_dir}...") - with tempfile.TemporaryDirectory() as tmp_dir: - try: - clone_git_repository( - url=repo_url, to_path=tmp_dir, commit=commit - ) - except RuntimeError: - error( - f"Could not find commit '{commit}' in repository " - f"'{repo_url}' of plugin '{display_name}'. This might " - "happen if the owner of the plugin has force-pushed to the " - "plugin repository or taken it down. Please report this " - "plugin version in the ZenML Hub or via Slack." - ) - plugin_dir = os.path.join(tmp_dir, subdir or "") - shutil.move(plugin_dir, output_dir) - declare(f"Successfully Cloned plugin '{display_name}'.") - - -@hub.command("login") -@click.option( - "--github", - "-g", - is_flag=True, - help="Login via GitHub.", -) -@click.option( - "--email", - "-e", - type=str, - help="Login via ZenML Hub account using this email address.", -) -@click.option( - "--password", - "-p", - type=str, - help="Password of the ZenML Hub account.", -) -def login( - github: bool = False, - email: Optional[str] = None, - password: Optional[str] = None, -) -> None: - """Login to the ZenML Hub. - - Args: - github: Login via GitHub. - email: Login via ZenML Hub account using this email address. - password: Password of the ZenML Hub account. Only used if `email` is - specified. - """ - if github: - _login_via_github() - elif email: - if not password: - password = click.prompt("Password", type=str, hide_input=True) - _login_via_zenml_hub(email, password) - else: - declare( - "You can either login via your ZenML Hub account or via GitHub." - ) - confirmation = click.confirm("Login via ZenML Hub account?") - if confirmation: - _login_via_zenml_hub() - else: - _login_via_github() - - -def _login_via_zenml_hub( - email: Optional[str] = None, password: Optional[str] = None -) -> None: - """Login via ZenML Hub email and password. - - Args: - email: Login via ZenML Hub account using this email address. - password: Password of the ZenML Hub account. Only used if `email` is - specified. - """ - client = HubClient() - if not email or not password: - declare("Please enter your ZenML Hub credentials.") - while not email: - email = click.prompt("Email", type=str) - while not password: - password = click.prompt("Password", type=str, hide_input=True) - try: - client.login(email, password) - me = client.get_me() - if me: - declare(f"Successfully logged in as: {me.username} ({me.email})!") - return - error("Could not retrieve user information from the ZenML Hub.") - except HubAPIError as e: - error(f"Could not login to the ZenML Hub: {e}") - - -def _login_via_github() -> None: - """Login via GitHub.""" - client = HubClient() - try: - login_url = client.get_github_login_url() - except HubAPIError as e: - error(f"Could not retrieve GitHub login URL: {e}") - declare(f"Please open the following URL in your browser: {login_url}") - auth_token = click.prompt("Please enter your auth token", type=str) - client.set_auth_token(auth_token) - declare("Successfully logged in to the ZenML Hub.") - - -@hub.command("logout") -def logout() -> None: - """Logout from the ZenML Hub.""" - client = HubClient() - client.set_auth_token(None) - declare("Successfully logged out from the ZenML Hub.") - - -@hub.command("submit") -@click.option( - "--plugin_name", - "-n", - type=str, - help=( - "Name of the plugin to submit. If not provided, the name will be asked " - "for interactively." - ), -) -@click.option( - "--version", - "-v", - type=str, - help=( - "Version of the plugin to submit. Can only be set if the plugin " - "already exists. If not provided, the version will be auto-incremented." - ), -) -@click.option( - "--release_notes", - "-r", - type=str, - help="Release notes for the plugin version.", -) -@click.option( - "--description", - "-d", - type=str, - help="Description of the plugin.", -) -@click.option( - "--repository_url", - "-u", - type=str, - help="URL to the public Git repository containing the plugin source code.", -) -@click.option( - "--repository_subdir", - "-s", - type=str, - help="Subdirectory of the repository containing the plugin source code.", -) -@click.option( - "--repository_branch", - "-b", - type=str, - help="Branch to checkout from the repository.", -) -@click.option( - "--repository_commit", - "-c", - type=str, - help="Commit to checkout from the repository. Overrides --branch.", -) -@click.option( - "tags", - "--tag", - "-t", - type=str, - multiple=True, -) -@click.option( - "--interactive", - "-i", - is_flag=True, - help="Run the command in interactive mode.", -) -def submit_plugin( - plugin_name: Optional[str] = None, - version: Optional[str] = None, - release_notes: Optional[str] = None, - description: Optional[str] = None, - repository_url: Optional[str] = None, - repository_subdir: Optional[str] = None, - repository_branch: Optional[str] = None, - repository_commit: Optional[str] = None, - tags: Optional[List[str]] = None, - interactive: bool = False, -) -> None: - """Submit a plugin to the ZenML Hub. - - Args: - plugin_name: Name of the plugin to submit. Needs to be set unless - interactive mode is enabled. - version: Version of the plugin to submit. Can only be set if the plugin - already exists. If not provided, the version will be - auto-incremented. - release_notes: Release notes for the plugin version. - description: Description of the plugin. - repository_url: URL to the public Git repository containing the plugin - source code. Needs to be set unless interactive mode is enabled. - repository_subdir: Subdirectory of the repository containing the plugin - source code. - repository_branch: Branch to checkout from the repository. - repository_commit: Commit to checkout from the repository. Overrides - `repository_branch`. - tags: Tags to add to the plugin. - interactive: Whether to run the command in interactive mode, asking for - missing or invalid parameters. - """ - with track_handler( - event=AnalyticsEvent.ZENML_HUB_PLUGIN_SUBMIT, - ) as analytics_handler: - client = HubClient() - analytics_handler.metadata["hub_url"] = client.url - - # Validate that the user is logged in - if not client.auth_token: - error( - "You must be logged in to contribute a plugin to the Hub. " - "Please run `zenml hub login` to login." - ) - me = client.get_me() - if not me: - error("Could not retrieve user information from the ZenML Hub.") - if not me.username: - error( - "Your ZenML Hub account does not have a username yet. Please " - "set a username in your account settings and try again." - ) - username = me.username - - # Validate the plugin name and check if it exists - plugin_name, plugin_exists = _validate_plugin_name( - client=client, - plugin_name=plugin_name, - username=username, - interactive=interactive, - ) - - # If the plugin exists, ask for version and release notes in - # interactive mode. - if plugin_exists and interactive: - if not version: - declare( - "You are about to create a new version of plugin " - f"'{plugin_name}'. By default, this will increment the " - "minor version of the plugin. If you want to specify a " - "different version, you can do so below. In that case, " - "make sure that the version is of shape '.' and " - "is higher than the current latest version of the plugin." - ) - version = click.prompt("(Optional) plugin version", default="") - if not release_notes: - declare( - f"You are about to create a new version of plugin " - f"'{plugin_name}'. You can optionally provide release " - "notes for this version below." - ) - release_notes = click.prompt( - "(Optional) release notes", default="" - ) - - # Clone the repo and validate the commit / branch / subdir / structure - ( - repository_url, - repository_commit, - repository_branch, - repository_subdir, - ) = _validate_repository( - url=repository_url, - commit=repository_commit, - branch=repository_branch, - subdir=repository_subdir, - interactive=interactive, - ) - - # In interactive mode, ask for a description if none is provided - if interactive and not description: - declare( - "You can optionally provide a description for your plugin " - "below. If not set, the first line of your README.md will " - "be used." - ) - description = click.prompt( - "(Optional) plugin description", default="" - ) - - # Validate the tags - if tags: - tags = _validate_tags(tags=tags, interactive=interactive) - else: - tags = [] - - # Make a create request to the hub - plugin_request = HubPluginRequestModel( - name=plugin_name, - description=description, - version=version, - release_notes=release_notes, - repository_url=repository_url, - repository_subdirectory=repository_subdir, - repository_branch=repository_branch, - repository_commit=repository_commit, - tags=tags, - ) - plugin_response = client.create_plugin( - plugin_request=plugin_request, - ) - - # Stream the build logs - plugin_name = plugin_response.name - plugin_version = plugin_response.version - declare( - "Thanks for submitting your plugin to the ZenML Hub. The plugin is " - "now being built into an installable package. This may take a few " - "minutes. To view the build logs, run " - f"`zenml hub logs {username}/{plugin_name}:{plugin_version}`." - ) - - -def _validate_plugin_name( - client: HubClient, - plugin_name: Optional[str], - username: str, - interactive: bool, -) -> Tuple[str, bool]: - """Validate that the plugin name is provided and available. - - Args: - client: The Hub client used to check if the plugin name is available. - plugin_name: The plugin name to validate. - username: The username of the current user. - interactive: Whether to run in interactive mode. - - Returns: - The validated plugin name, and whether the plugin already exists. - """ - # Make sure the plugin name is provided. - while not plugin_name: - if not interactive: - error("Plugin name not provided.") - declare("Please enter a name for the plugin.") - plugin_name = click.prompt("Plugin name") - - existing_plugin = client.get_plugin(name=plugin_name, author=username) - return plugin_name, bool(existing_plugin) - - -def _validate_repository( - url: Optional[str], - commit: Optional[str], - branch: Optional[str], - subdir: Optional[str], - interactive: bool, -) -> Tuple[str, Optional[str], Optional[str], Optional[str]]: - """Validate the provided repository arguments. - - Args: - url: The URL to the repository to clone. - commit: The commit to checkout. - branch: The branch to checkout. Will be ignored if commit is provided. - subdir: The subdirectory in which the plugin is located. - interactive: Whether to run in interactive mode. - - Returns: - The validated URL, commit, branch, and subdirectory. - """ - from zenml.utils.git_utils import clone_git_repository - - while True: - # Make sure the repository URL is provided. - if not url: - if not interactive: - error("Repository URL not provided.") - declare( - "Please enter the URL to the public Git repository containing " - "the plugin source code." - ) - url = click.prompt("Repository URL") - assert url is not None - - # In interactive mode, ask for the branch and commit if not provided - if interactive and not branch and not commit: - confirmation = click.confirm( - "Do you want to use the latest commit from the 'main' branch " - "of the repository?" - ) - if not confirmation: - confirmation = click.confirm( - "You can either use a specific commit or the latest commit " - "from one of your branches. Do you want to use a specific " - "commit?" - ) - if confirmation: - declare("Please enter the SHA of the commit.") - commit = click.prompt("Repository commit") - branch = None - else: - declare("Please enter the name of a branch.") - branch = click.prompt("Repository branch") - commit = None - - try: - # Check if the URL/branch/commit are valid. - with tempfile.TemporaryDirectory() as tmp_dir: - clone_git_repository( - url=url, - commit=commit, - branch=branch, - to_path=tmp_dir, - ) - - # Check if the subdir exists and has the correct structure. - subdir = _validate_repository_subdir( - repository_subdir=subdir, - repo_path=tmp_dir, - interactive=interactive, - ) - - return url, commit, branch, subdir - - except RuntimeError: - repo_display_name = f"'{url}'" - suggestion = "Please enter a valid repository URL" - if commit: - repo_display_name += f" (commit '{commit}')" - suggestion += " and make sure the commit exists." - elif branch: - repo_display_name += f" (branch '{branch}')" - suggestion += " and make sure the branch exists." - else: - suggestion += " and make sure the 'main' branch exists." - msg = f"Could not clone repository from URL {repo_display_name}. " - if not interactive: - error(msg + suggestion) - declare(msg + suggestion) - url, branch, commit = None, None, None - - -def _validate_repository_subdir( - repository_subdir: Optional[str], repo_path: str, interactive: bool -) -> Optional[str]: - """Validate the provided repository subdirectory. - - Args: - repository_subdir: The subdirectory to validate. - repo_path: The path to the repository to validate the subdirectory in. - interactive: Whether to run in interactive mode. - - Returns: - The validated subdirectory. - """ - while True: - # In interactive mode, ask for the subdirectory if not provided - if interactive and not repository_subdir: - confirmation = click.confirm( - "Is the plugin source code in the root of the repository?" - ) - if not confirmation: - declare( - "Please enter the subdirectory of the repository " - "containing the plugin source code." - ) - repository_subdir = click.prompt("Repository subdirectory") - - # If a subdir was provided, make sure it exists - if repository_subdir: - subdir_path = os.path.join(repo_path, repository_subdir) - if not os.path.exists(subdir_path): - if not interactive: - error("Repository subdirectory does not exist.") - declare( - f"Subdirectory '{repository_subdir}' does not exist in the " - f"repository." - ) - declare("Please enter a valid subdirectory.") - repository_subdir = click.prompt( - "Repository subdirectory", default="" - ) - continue - - # Check if the plugin structure is valid. - if repository_subdir: - plugin_path = os.path.join(repo_path, repository_subdir) - else: - plugin_path = repo_path - try: - _validate_repository_structure(plugin_path) - return repository_subdir - except ValueError as e: - msg = ( - f"Plugin code structure at {repository_subdir} is invalid: " - f"{str(e)}" - ) - if not interactive: - error(str(e)) - declare(msg) - declare("Please enter a valid subdirectory.") - repository_subdir = click.prompt("Repository subdirectory") - - -def _validate_repository_structure(plugin_root: str) -> None: - """Validate the repository structure of a submitted ZenML Hub plugin. - - We expect the following structure: - - src/__init__.py - - README.md - - requirements.txt - - (Optional) logo.png - - Args: - plugin_root: Root directory of the plugin. - - Raises: - ValueError: If the repo does not have the correct structure. - """ - # src/__init__.py exists. - src_path = os.path.join(plugin_root, "src") - if not os.path.exists(src_path): - raise ValueError("src/ not found") - init_path = os.path.join(src_path, "__init__.py") - if not os.path.exists(init_path): - raise ValueError("src/__init__.py not found") - - # README.md exists. - readme_path = os.path.join(plugin_root, "README.md") - if not os.path.exists(readme_path): - raise ValueError("README.md not found") - - # requirements.txt exists. - requirements_path = os.path.join(plugin_root, "requirements.txt") - if not os.path.exists(requirements_path): - raise ValueError("requirements.txt not found") - - -def _validate_tags(tags: List[str], interactive: bool) -> List[str]: - """Validate the provided tags. - - Args: - tags: The tags to validate. - interactive: Whether to run in interactive mode. - - Returns: - The validated tags. - """ - if not tags: - if not interactive: - return [] - - # In interactive mode, ask for tags if none were provided. - return _ask_for_tags() - - # If tags were provided, print a warning if any of them is invalid. - for tag in tags: - if tag.startswith(ZENML_HUB_INTERNAL_TAG_PREFIX): - warning( - f"Tag '{tag}' will be ignored because it starts with " - f"disallowed prefix '{ZENML_HUB_INTERNAL_TAG_PREFIX}'." - ) - return tags - - -def _ask_for_tags() -> List[str]: - """Repeatedly ask the user for tags to assign to the plugin. - - Returns: - A list of tags. - """ - tags: List[str] = [] - while True: - tag = click.prompt( - "(Optional) enter tags you want to assign to the plugin.", - default="", - ) - if not tag: - return tags - if tag.startswith(ZENML_HUB_INTERNAL_TAG_PREFIX): - warning( - "User-defined tags may not start with " - f"'{ZENML_HUB_INTERNAL_TAG_PREFIX}'." - ) - else: - tags.append(tag) - - -@hub.command("submit-batch") -@click.argument( - "config", type=click.Path(exists=True, dir_okay=False), required=True -) -def batch_submit(config: str) -> None: - """Batch submit plugins to the ZenML Hub. - - WARNING: This command is intended for advanced users only. It does not - perform any client-side validation which might lead to server-side HTTP - errors that are hard to debug if the config file you specify is invalid or - contains invalid plugin definitions. When in doubt, use the - `zenml hub submit` command instead. - - Args: - config: Path to the config file. The config file is expected to be a - list of plugin definitions. Each plugin definition must be a dict - with keys and values matching the fields of `HubPluginRequestModel`: - ```yaml - - name: str - version: str - release_notes: str - description: str - repository_url: str - repository_subdirectory: str - repository_branch: str - repository_commit: str - logo_url: str - tags: - - str - - ... - - ... - ``` - """ - from pydantic import ValidationError - - from zenml.utils.yaml_utils import read_yaml - - client = HubClient() - config = read_yaml(config) - if not isinstance(config, list): - error("Config file must be a list of plugin definitions.") - declare(f"Submitting {len(config)} plugins to the hub...") - for plugin_dict in config: - try: - assert isinstance(plugin_dict, dict) - plugin_request = HubPluginRequestModel(**plugin_dict) - plugin = client.create_plugin(plugin_request=plugin_request) - except (AssertionError, ValidationError, HubAPIError) as e: - warning(f"Could not submit plugin: {str(e)}") - continue - display_name = plugin_display_name( - name=plugin.name, - version=plugin.version, - author=plugin.author, - ) - declare(f"Submitted plugin '{display_name}' to the hub.") - - -@hub.command("logs") -@click.argument("plugin_name", type=str, required=True) -def get_logs(plugin_name: str) -> None: - """Get the build logs of a ZenML Hub plugin. - - Args: - plugin_name: Name of the plugin. - """ - client = HubClient() - author, plugin_name, plugin_version = parse_plugin_name(plugin_name) - display_name = plugin_display_name(plugin_name, plugin_version, author) - - # Get the plugin from the hub - plugin = client.get_plugin( - name=plugin_name, - version=plugin_version, - author=author, - ) - if not plugin: - error(f"Could not find plugin '{display_name}' on the hub.") - - if plugin.status == PluginStatus.PENDING: - error( - f"Plugin '{display_name}' is still being built. Please try " - "again later." - ) - - if not plugin.build_logs: - declare( - f"Plugin '{display_name}' finished building, but no logs " - "were found." - ) - return - - for line in plugin.build_logs.splitlines(): - declare(line) - - -# GENERAL HELPER FUNCTIONS - - -def _is_plugin_installed(author: str, plugin_name: str) -> bool: - """Helper function to check if a plugin is installed. - - Args: - author: The author of the plugin. - plugin_name: The name of the plugin. - - Returns: - Whether the plugin is installed. - """ - module_name = _get_plugin_module(author=author, plugin_name=plugin_name) - try: - spec = find_spec(module_name) - return spec is not None - except ModuleNotFoundError: - return False - - -def _get_plugin_module(author: str, plugin_name: str) -> str: - """Helper function to get the module name of a plugin. - - Args: - author: The author of the plugin. - plugin_name: The name of the plugin. - - Returns: - The module name of the plugin. - """ - module_name = "zenml.hub" - if author != ZENML_HUB_ADMIN_USERNAME: - module_name += f".{author}" - module_name += f".{plugin_name}" - return module_name diff --git a/src/zenml/config/docker_settings.py b/src/zenml/config/docker_settings.py index 083d564fdd5..2d86cf33b33 100644 --- a/src/zenml/config/docker_settings.py +++ b/src/zenml/config/docker_settings.py @@ -102,7 +102,6 @@ class DockerSettings(BaseSettings): Depending on the configuration of this object, requirements will be installed in the following order (each step optional): - The packages installed in your local python environment - - The packages specified via the `required_hub_plugins` attribute - The packages required by the stack unless this is disabled by setting `install_stack_requirements=False`. - The packages specified via the `required_integrations` @@ -161,11 +160,7 @@ class DockerSettings(BaseSettings): required_integrations: List of ZenML integrations that should be installed. All requirements for the specified integrations will be installed inside the Docker image. - required_hub_plugins: List of ZenML Hub plugins to install. - Expected format: '(/)=='. - If no version is specified, the latest version is taken. The - packages of required plugins and all their dependencies will be - installed inside the Docker image. + required_hub_plugins: DEPRECATED/UNUSED. install_stack_requirements: If `True`, ZenML will automatically detect if components of your active stack are part of a ZenML integration and install the corresponding requirements and apt packages. @@ -228,7 +223,7 @@ class DockerSettings(BaseSettings): source_files: SourceFileMode = SourceFileMode.DOWNLOAD_OR_INCLUDE _deprecation_validator = deprecation_utils.deprecate_pydantic_attributes( - "copy_files", "copy_global_config" + "copy_files", "copy_global_config", "required_hub_plugins" ) @model_validator(mode="before") diff --git a/src/zenml/constants.py b/src/zenml/constants.py index c27adf566b3..05b5152aae1 100644 --- a/src/zenml/constants.py +++ b/src/zenml/constants.py @@ -165,7 +165,6 @@ def handle_int_env_var(var: str, default: int = 0) -> int: ENV_ZENML_REQUIRES_CODE_DOWNLOAD = "ZENML_REQUIRES_CODE_DOWNLOAD" ENV_ZENML_SERVER = "ZENML_SERVER" ENV_ZENML_LOCAL_SERVER = "ZENML_LOCAL_SERVER" -ENV_ZENML_HUB_URL = "ZENML_HUB_URL" ENV_ZENML_ENFORCE_TYPE_ANNOTATIONS = "ZENML_ENFORCE_TYPE_ANNOTATIONS" ENV_ZENML_ENABLE_IMPLICIT_AUTH_METHODS = "ZENML_ENABLE_IMPLICIT_AUTH_METHODS" ENV_ZENML_DISABLE_STEP_LOGS_STORAGE = "ZENML_DISABLE_STEP_LOGS_STORAGE" @@ -279,7 +278,6 @@ def handle_int_env_var(var: str, default: int = 0) -> int: _csp_script_src_urls = ["https://widgets-v3.featureos.app"] _csp_connect_src_urls = [ "https://sdkdocs.zenml.io", - "https://hubapi.zenml.io", "https://analytics.zenml.io", ] _csp_img_src_urls = [ diff --git a/src/zenml/enums.py b/src/zenml/enums.py index 2df9e6af793..79636392e79 100644 --- a/src/zenml/enums.py +++ b/src/zenml/enums.py @@ -171,7 +171,6 @@ class CliCategories(StrEnum): STACK_COMPONENTS = "Stack Components" MODEL_DEPLOYMENT = "Model Deployment" - HUB = "ZenML Hub" INTEGRATIONS = "Integrations" MANAGEMENT_TOOLS = "Management Tools" MODEL_CONTROL_PLANE = "Model Control Plane" diff --git a/src/zenml/logger.py b/src/zenml/logger.py index 1883711345e..c8036f3038f 100644 --- a/src/zenml/logger.py +++ b/src/zenml/logger.py @@ -97,7 +97,7 @@ def format(self, record: logging.LogRecord) -> str: ) # Format URLs - url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' + url_pattern = r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" urls = re.findall(url_pattern, formatted_message) for url in urls: formatted_message = formatted_message.replace( diff --git a/src/zenml/models/__init__.py b/src/zenml/models/__init__.py index 989d0b865f3..7eef7b6fd97 100644 --- a/src/zenml/models/__init__.py +++ b/src/zenml/models/__init__.py @@ -362,13 +362,6 @@ from zenml.models.v2.misc.user_auth import UserAuthModel from zenml.models.v2.misc.build_item import BuildItem from zenml.models.v2.misc.loaded_visualization import LoadedVisualization -from zenml.models.v2.misc.hub_plugin_models import ( - HubPluginRequestModel, - HubPluginResponseModel, - HubUserResponseModel, - HubPluginBaseModel, - PluginStatus, -) from zenml.models.v2.misc.external_user import ExternalUserModel from zenml.models.v2.misc.auth_models import ( OAuthDeviceAuthorizationRequest, @@ -735,11 +728,6 @@ "ExternalUserModel", "BuildItem", "LoadedVisualization", - "HubPluginRequestModel", - "HubPluginResponseModel", - "HubUserResponseModel", - "HubPluginBaseModel", - "PluginStatus", "ServerModel", "ServerDatabaseType", "ServerDeploymentType", diff --git a/src/zenml/models/v2/core/user.py b/src/zenml/models/v2/core/user.py index 5b72c1d7451..5d3053914b0 100644 --- a/src/zenml/models/v2/core/user.py +++ b/src/zenml/models/v2/core/user.py @@ -65,12 +65,6 @@ class UserBase(BaseModel): description="`null` if not answered, `true` if agreed, " "`false` if skipped.", ) - hub_token: Optional[str] = Field( - default=None, - title="JWT Token for the connected Hub account. Only relevant for user " - "accounts.", - max_length=STR_FIELD_MAX_LENGTH, - ) password: Optional[str] = Field( default=None, title="A password for the user.", @@ -296,12 +290,6 @@ class UserResponseMetadata(BaseResponseMetadata): "for user accounts.", max_length=STR_FIELD_MAX_LENGTH, ) - hub_token: Optional[str] = Field( - default=None, - title="JWT Token for the connected Hub account. Only relevant for user " - "accounts.", - max_length=STR_FIELD_MAX_LENGTH, - ) external_user_id: Optional[UUID] = Field( default=None, title="The external user ID associated with the account. Only relevant " @@ -416,15 +404,6 @@ def email(self) -> Optional[str]: """ return self.get_metadata().email - @property - def hub_token(self) -> Optional[str]: - """The `hub_token` property. - - Returns: - the value of the property. - """ - return self.get_metadata().hub_token - @property def external_user_id(self) -> Optional[UUID]: """The `external_user_id` property. diff --git a/src/zenml/models/v2/misc/hub_plugin_models.py b/src/zenml/models/v2/misc/hub_plugin_models.py deleted file mode 100644 index 8523024d5e4..00000000000 --- a/src/zenml/models/v2/misc/hub_plugin_models.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) ZenML GmbH 2023. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing -# permissions and limitations under the License. -"""Models representing ZenML Hub plugins.""" - -from datetime import datetime -from typing import List, Optional -from uuid import UUID - -from pydantic import BaseModel - -from zenml.utils.enum_utils import StrEnum - - -class PluginStatus(StrEnum): - """Enum that represents the status of a plugin. - - - PENDING: Plugin is being built - - FAILED: Plugin build failed - - AVAILABLE: Plugin is available for installation - - YANKED: Plugin was yanked and is no longer available - """ - - PENDING = "pending" - FAILED = "failed" - AVAILABLE = "available" - YANKED = "yanked" - - -class HubUserResponseModel(BaseModel): - """Model for a ZenML Hub user.""" - - id: UUID - email: str - username: Optional[str] = None - - -class HubPluginBaseModel(BaseModel): - """Base model for a ZenML Hub plugin.""" - - name: str - description: Optional[str] = None - version: Optional[str] = None - release_notes: Optional[str] = None - repository_url: str - repository_subdirectory: Optional[str] = None - repository_branch: Optional[str] = None - repository_commit: Optional[str] = None - tags: Optional[List[str]] = None - logo_url: Optional[str] = None - - -class HubPluginRequestModel(HubPluginBaseModel): - """Request model for a ZenML Hub plugin.""" - - -class HubPluginResponseModel(HubPluginBaseModel): - """Response model for a ZenML Hub plugin.""" - - id: UUID - status: PluginStatus - author: str - version: str - index_url: Optional[str] = None - package_name: Optional[str] = None - requirements: Optional[List[str]] = None - build_logs: Optional[str] = None - created: datetime - updated: datetime diff --git a/src/zenml/utils/pipeline_docker_image_builder.py b/src/zenml/utils/pipeline_docker_image_builder.py index d5c43c9e5bc..16081b6486e 100644 --- a/src/zenml/utils/pipeline_docker_image_builder.py +++ b/src/zenml/utils/pipeline_docker_image_builder.py @@ -17,11 +17,9 @@ import os import subprocess import sys -from collections import defaultdict from typing import ( TYPE_CHECKING, Any, - DefaultDict, Dict, List, Optional, @@ -444,8 +442,9 @@ def gather_requirements_files( requirements files. The files will be in the following order: - Packages installed in the local Python environment + - Requirements defined by stack integrations + - Requirements defined by user integrations - User-defined requirements - - Requirements defined by user-defined and/or stack integrations """ requirements_files: List[Tuple[str, str, List[str]]] = [] @@ -481,43 +480,6 @@ def gather_requirements_files( "- Including python packages from local environment" ) - # Generate requirements files for all ZenML Hub plugins - if docker_settings.required_hub_plugins: - ( - hub_internal_requirements, - hub_pypi_requirements, - ) = PipelineDockerImageBuilder._get_hub_requirements( - docker_settings.required_hub_plugins - ) - - # Plugin packages themselves - for i, (index, packages) in enumerate( - hub_internal_requirements.items() - ): - file_name = f".zenml_hub_internal_requirements_{i}" - file_lines = [f"-i {index}", *packages] - file_contents = "\n".join(file_lines) - requirements_files.append( - (file_name, file_contents, ["--no-deps"]) - ) - if log: - logger.info( - "- Including internal hub packages from index `%s`: %s", - index, - ", ".join(f"`{r}`" for r in packages), - ) - - # PyPI requirements of plugin packages - if hub_pypi_requirements: - file_name = ".zenml_hub_pypi_requirements" - file_contents = "\n".join(hub_pypi_requirements) - requirements_files.append((file_name, file_contents, [])) - if log: - logger.info( - "- Including hub requirements from PyPI: %s", - ", ".join(f"`{r}`" for r in hub_pypi_requirements), - ) - if docker_settings.install_stack_requirements: stack_requirements = stack.requirements() if code_repository: @@ -599,59 +561,6 @@ def gather_requirements_files( return requirements_files - @staticmethod - def _get_hub_requirements( - required_hub_plugins: List[str], - ) -> Tuple[Dict[str, List[str]], List[str]]: - """Get package requirements for ZenML Hub plugins. - - Args: - required_hub_plugins: List of hub plugin names in the format - `(/)(==)`. - - Returns: - - A dict of the hub plugin packages themselves (which need to be - installed from a custom index, mapping index URLs to lists of - package names. - - A list of all unique dependencies of the required hub plugins - (which can be installed from PyPI). - """ - from zenml._hub.client import HubClient - from zenml._hub.utils import parse_plugin_name, plugin_display_name - - client = HubClient() - - internal_requirements: DefaultDict[str, List[str]] = defaultdict(list) - pypi_requirements: List[str] = [] - - for plugin_str in required_hub_plugins: - author, name, version = parse_plugin_name( - plugin_str, version_separator="==" - ) - - plugin = client.get_plugin( - name=name, - version=version, - author=author, - ) - - if plugin and plugin.index_url and plugin.package_name: - internal_requirements[plugin.index_url].append( - plugin.package_name - ) - if plugin.requirements: - pypi_requirements.extend(plugin.requirements) - else: - display_name = plugin_display_name(name, version, author) - logger.warning( - "Hub plugin `%s` does not exist or cannot be installed." - "Skipping installation of this plugin.", - display_name, - ) - - pypi_requirements = sorted(set(pypi_requirements)) - return dict(internal_requirements), pypi_requirements - @staticmethod def _generate_zenml_pipeline_dockerfile( parent_image: str, diff --git a/src/zenml/zen_server/routers/users_endpoints.py b/src/zenml/zen_server/routers/users_endpoints.py index 4dfe8f14bb3..b2e6422bacc 100644 --- a/src/zenml/zen_server/routers/users_endpoints.py +++ b/src/zenml/zen_server/routers/users_endpoints.py @@ -286,7 +286,6 @@ def update_user( # - active # - password # - email_opted_in + email - # - hub_token # safe_user_update = user_update.create_copy( exclude={ @@ -298,7 +297,6 @@ def update_user( "old_password", "email_opted_in", "email", - "hub_token", }, ) @@ -387,7 +385,6 @@ def update_user( if ( user_update.email_opted_in is not None or user_update.email is not None - or user_update.hub_token is not None ): if user.id != auth_context.user.id: raise IllegalOperationError( @@ -399,8 +396,6 @@ def update_user( if safe_user_update.email_opted_in is not None: safe_user_update.email_opted_in = user_update.email_opted_in safe_user_update.email = user_update.email - if safe_user_update.hub_token is not None: - safe_user_update.hub_token = user_update.hub_token updated_user = zen_store().update_user( user_id=user.id, @@ -444,7 +439,6 @@ def activate_user( # - is_admin # - active # - old_password - # - hub_token # safe_user_update = user_update.create_copy( exclude={ @@ -453,7 +447,6 @@ def activate_user( "is_admin", "active", "old_password", - "hub_token", }, ) diff --git a/src/zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py b/src/zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py new file mode 100644 index 00000000000..191b767f2b0 --- /dev/null +++ b/src/zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py @@ -0,0 +1,36 @@ +"""Remove user hub token [909550c7c4da]. + +Revision ID: 909550c7c4da +Revises: 0.63.0 +Create Date: 2024-08-05 16:02:48.990897 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "909550c7c4da" +down_revision = "0.63.0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Upgrade database schema and/or data, creating a new revision.""" + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("user", schema=None) as batch_op: + batch_op.drop_column("hub_token") + + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade database schema and/or data back to the previous revision.""" + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("user", schema=None) as batch_op: + batch_op.add_column( + sa.Column("hub_token", sa.VARCHAR(), nullable=True) + ) + + # ### end Alembic commands ### diff --git a/src/zenml/zen_stores/schemas/user_schemas.py b/src/zenml/zen_stores/schemas/user_schemas.py index 3278f8aa852..f4482e464c3 100644 --- a/src/zenml/zen_stores/schemas/user_schemas.py +++ b/src/zenml/zen_stores/schemas/user_schemas.py @@ -77,7 +77,6 @@ class UserSchema(NamedSchema, table=True): active: bool password: Optional[str] = Field(nullable=True) activation_token: Optional[str] = Field(nullable=True) - hub_token: Optional[str] = Field(nullable=True) email_opted_in: Optional[bool] = Field(nullable=True) external_user_id: Optional[UUID] = Field(nullable=True) is_admin: bool = Field(default=False) diff --git a/tests/unit/_hub/test_client.py b/tests/unit/_hub/test_client.py deleted file mode 100644 index 25cbe976ebc..00000000000 --- a/tests/unit/_hub/test_client.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) ZenML GmbH 2023. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing -# permissions and limitations under the License. -"""Unit tests for the ZenML Hub client.""" - -from zenml._hub.client import HubClient -from zenml._hub.constants import ZENML_HUB_DEFAULT_URL -from zenml.constants import ENV_ZENML_HUB_URL - - -def test_default_url(mocker): - """Test that the default URL is set correctly.""" - client = HubClient() - assert client.url == ZENML_HUB_DEFAULT_URL - - # Pass a URL to the constructor. - client = HubClient(url="test_url") - assert client.url == "test_url" - - # Mock setting the environment variable. - mocker.patch.dict("os.environ", {ENV_ZENML_HUB_URL: "test_url"}) - client = HubClient() - assert client.url == "test_url" - - -def test_list_plugins(): - """Test listing plugins.""" - client = HubClient() - plugins = client.list_plugins() - assert len(plugins) > 0 - - -def test_get_plugin(): - """Test getting a plugin.""" - plugin_name = "langchain_qa_example" - client = HubClient() - plugin = client.get_plugin(plugin_name) - assert plugin.name == plugin_name - - # Test getting a specific version. - version = "0.1" - plugin = client.get_plugin(plugin_name, version=version) - assert plugin.name == plugin_name - assert plugin.version == version - - # Test getting a non-existent plugin. - plugin_name = "non_existent_plugin_by_aria_and_blupus" - client = HubClient() - plugin = client.get_plugin(plugin_name) - assert plugin is None diff --git a/tests/unit/_hub/test_utils.py b/tests/unit/_hub/test_utils.py deleted file mode 100644 index 8d563f08540..00000000000 --- a/tests/unit/_hub/test_utils.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) ZenML GmbH 2023. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing -# permissions and limitations under the License. -"""Unit tests for zenml._hub.utils.py""" - -import pytest - -from zenml._hub.utils import parse_plugin_name, plugin_display_name - - -@pytest.mark.parametrize( - "plugin_name, author, name, version", - [ - ("author/plugin_name:version", "author", "plugin_name", "version"), - ("author/plugin_name", "author", "plugin_name", "latest"), - ("plugin_name:version", None, "plugin_name", "version"), - ("plugin_name", None, "plugin_name", "latest"), - ], -) -def test_parse_plugin_name(plugin_name, author, name, version): - """Unit test for `parse_plugin_name`.""" - assert parse_plugin_name(plugin_name) == (author, name, version) - - # Test with different separators. - plugin_name_2 = name - if author: - plugin_name_2 = f"{author},{plugin_name_2}" - if version: - plugin_name_2 = f"{plugin_name_2};{version}" - author_2, name_2, version_2 = parse_plugin_name( - plugin_name_2, - author_separator=",", - version_separator=";", - ) - print(plugin_name, plugin_name_2) - print(author, author_2) - print(name, name_2) - print(version, version_2) - assert author_2 == author - assert name_2 == name - assert version_2 == version - - -@pytest.mark.parametrize( - "invalid_plugin_name", - [ - "", - "invalid/plugin/name", - "invalid:plugin:name", - ], -) -def test_parse_invalid_plugin_name(invalid_plugin_name): - """Unit test for `parse_plugin_name`.""" - with pytest.raises(ValueError): - parse_plugin_name(invalid_plugin_name) - - -@pytest.mark.parametrize( - "plugin_name, author, name, version", - [ - ("author/plugin_name:version", "author", "plugin_name", "version"), - ("author/plugin_name:latest", "author", "plugin_name", None), - ("plugin_name:version", None, "plugin_name", "version"), - ("plugin_name:latest", None, "plugin_name", None), - ], -) -def test_plugin_display_name(plugin_name, author, name, version): - """Unit test for `plugin_display_name`.""" - assert plugin_display_name(name, version, author) == plugin_name From be984ed2b293975c9be3919226119d88edcc7a77 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 5 Aug 2024 16:06:22 +0200 Subject: [PATCH 2/5] Remove more references --- .../autogenerate-a-template-yaml-file.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/book/how-to/use-configuration-files/autogenerate-a-template-yaml-file.md b/docs/book/how-to/use-configuration-files/autogenerate-a-template-yaml-file.md index b073e03ad95..bdda4f51809 100644 --- a/docs/book/how-to/use-configuration-files/autogenerate-a-template-yaml-file.md +++ b/docs/book/how-to/use-configuration-files/autogenerate-a-template-yaml-file.md @@ -70,7 +70,6 @@ settings: python_package_installer: PythonPackageInstaller replicate_local_python_environment: Union[List[str], PythonEnvironmentExportMethod, NoneType] - required_hub_plugins: List[str] required_integrations: List[str] requirements: Union[NoneType, str, List[str]] skip_build: bool @@ -131,7 +130,6 @@ steps: python_package_installer: PythonPackageInstaller replicate_local_python_environment: Union[List[str], PythonEnvironmentExportMethod, NoneType] - required_hub_plugins: List[str] required_integrations: List[str] requirements: Union[NoneType, str, List[str]] skip_build: bool @@ -190,7 +188,6 @@ steps: python_package_installer: PythonPackageInstaller replicate_local_python_environment: Union[List[str], PythonEnvironmentExportMethod, NoneType] - required_hub_plugins: List[str] required_integrations: List[str] requirements: Union[NoneType, str, List[str]] skip_build: bool From ed121e7ca95a0e5f99624315080b120cd86fc997 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 5 Aug 2024 19:58:09 +0200 Subject: [PATCH 3/5] Remove token access --- src/zenml/zen_stores/schemas/user_schemas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/zenml/zen_stores/schemas/user_schemas.py b/src/zenml/zen_stores/schemas/user_schemas.py index f4482e464c3..9f0c0f0cdd8 100644 --- a/src/zenml/zen_stores/schemas/user_schemas.py +++ b/src/zenml/zen_stores/schemas/user_schemas.py @@ -280,7 +280,6 @@ def to_model( if include_metadata: metadata = UserResponseMetadata( email=self.email if include_private else None, - hub_token=self.hub_token if include_private else None, external_user_id=self.external_user_id, user_metadata=json.loads(self.user_metadata) if self.user_metadata From 28e70c162d64fd1ea62d95bf08ae9cc2312a7231 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Tue, 6 Aug 2024 09:09:15 +0200 Subject: [PATCH 4/5] Remove hub references in test --- .../functional/zen_stores/test_zen_store.py | 1 - tests/unit/conftest.py | 19 ------------- .../test_pipeline_docker_image_builder.py | 27 ++++--------------- 3 files changed, 5 insertions(+), 42 deletions(-) diff --git a/tests/integration/functional/zen_stores/test_zen_store.py b/tests/integration/functional/zen_stores/test_zen_store.py index 57b18b8031c..b65f25856a5 100644 --- a/tests/integration/functional/zen_stores/test_zen_store.py +++ b/tests/integration/functional/zen_stores/test_zen_store.py @@ -1314,7 +1314,6 @@ def test_get_service_account(): assert user.is_service_account is True assert user.full_name == "" assert user.email_opted_in is False - assert user.hub_token is None # Get a service account as a user account by name with pytest.raises(KeyError): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 2c8b5fd6f1a..03377a29eb5 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -41,7 +41,6 @@ CodeRepositoryResponse, CodeRepositoryResponseBody, CodeRepositoryResponseMetadata, - HubPluginResponseModel, PipelineBuildResponse, PipelineBuildResponseBody, PipelineBuildResponseMetadata, @@ -56,7 +55,6 @@ PipelineRunResponse, PipelineRunResponseBody, PipelineRunResponseMetadata, - PluginStatus, StepRunRequest, StepRunResponse, StepRunResponseBody, @@ -682,23 +680,6 @@ def sample_code_repo_response_model( ) -@pytest.fixture -def sample_hub_plugin_response_model() -> HubPluginResponseModel: - return HubPluginResponseModel( - id=uuid4(), - author="AlexejPenner", - name="alexejs_ploogin", - version="3.14", - repository_url="https://github.com/zenml-io/zenml", - index_url="https://test.pypi.org/simple/", - package_name="ploogin", - status=PluginStatus.AVAILABLE, - created=datetime.now(), - updated=datetime.now(), - requirements=["ploogin==0.0.1", "zenml>=0.1.0"], - ) - - # Test data service_id = "12345678-1234-5678-1234-567812345678" service_name = "test_service" diff --git a/tests/unit/utils/test_pipeline_docker_image_builder.py b/tests/unit/utils/test_pipeline_docker_image_builder.py index 1753bbcfad6..171d4a3b9fe 100644 --- a/tests/unit/utils/test_pipeline_docker_image_builder.py +++ b/tests/unit/utils/test_pipeline_docker_image_builder.py @@ -47,18 +47,12 @@ def test_check_user_is_set(): assert "USER test_user" in generated_dockerfile -def test_requirements_file_generation( - mocker, local_stack, tmp_path: Path, sample_hub_plugin_response_model -): +def test_requirements_file_generation(mocker, local_stack, tmp_path: Path): """Tests that the requirements get included in the correct order and only when configured.""" mocker.patch("subprocess.check_output", return_value=b"local_requirements") mocker.patch.object( local_stack, "requirements", return_value={"stack_requirements"} ) - mocker.patch( - "zenml._hub.client.HubClient.get_plugin", - return_value=sample_hub_plugin_response_model, - ) # just local requirements settings = DockerSettings( @@ -106,34 +100,23 @@ def test_requirements_file_generation( install_stack_requirements=True, requirements=str(requirements_file), required_integrations=[SKLEARN], - required_hub_plugins=[sample_hub_plugin_response_model.name], replicate_local_python_environment="pip_freeze", ) files = PipelineDockerImageBuilder.gather_requirements_files( settings, stack=local_stack ) - assert len(files) == 6 + assert len(files) == 4 # first up the local python requirements assert files[0][1] == "local_requirements" - # then the hub requirements - expected_hub_internal_requirements = ( - f"-i {sample_hub_plugin_response_model.index_url}\n" - f"{sample_hub_plugin_response_model.package_name}" - ) - assert files[1][1] == expected_hub_internal_requirements - expected_hub_pypi_requirements = "\n".join( - sample_hub_plugin_response_model.requirements - ) - assert files[2][1] == expected_hub_pypi_requirements # then the stack requirements - assert files[3][1] == "stack_requirements" + assert files[1][1] == "stack_requirements" # then the integration requirements expected_integration_requirements = "\n".join( sorted(SklearnIntegration.REQUIREMENTS) ) - assert files[4][1] == expected_integration_requirements + assert files[2][1] == expected_integration_requirements # last the user requirements - assert files[5][1] == "user_requirements" + assert files[3][1] == "user_requirements" def test_build_skipping(): From f6dccf17f6ccbdf0bdb02227f260b937b1e6bbe1 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Tue, 6 Aug 2024 09:35:37 +0200 Subject: [PATCH 5/5] Remove from client --- src/zenml/client.py | 4 ---- src/zenml/models/v2/misc/user_auth.py | 7 ------- 2 files changed, 11 deletions(-) diff --git a/src/zenml/client.py b/src/zenml/client.py index 774c3046744..805f7690cc7 100644 --- a/src/zenml/client.py +++ b/src/zenml/client.py @@ -868,7 +868,6 @@ def update_user( updated_full_name: Optional[str] = None, updated_email: Optional[str] = None, updated_email_opt_in: Optional[bool] = None, - updated_hub_token: Optional[str] = None, updated_password: Optional[str] = None, old_password: Optional[str] = None, updated_is_admin: Optional[bool] = None, @@ -883,7 +882,6 @@ def update_user( updated_full_name: The new full name of the user. updated_email: The new email of the user. updated_email_opt_in: The new email opt-in status of the user. - updated_hub_token: Update the hub token updated_password: The new password of the user. old_password: The old password of the user. Required for password update. @@ -911,8 +909,6 @@ def update_user( ) if updated_email_opt_in is not None: user_update.email_opted_in = updated_email_opt_in - if updated_hub_token is not None: - user_update.hub_token = updated_hub_token if updated_password is not None: user_update.password = updated_password if old_password is None: diff --git a/src/zenml/models/v2/misc/user_auth.py b/src/zenml/models/v2/misc/user_auth.py index f69935420ec..a98566dcf2b 100644 --- a/src/zenml/models/v2/misc/user_auth.py +++ b/src/zenml/models/v2/misc/user_auth.py @@ -73,13 +73,6 @@ class UserAuthModel(BaseZenModel): "`false` if skipped.", ) - hub_token: Optional[str] = Field( - default=None, - title="JWT Token for the connected Hub account. Only relevant for user " - "accounts.", - max_length=STR_FIELD_MAX_LENGTH, - ) - @classmethod def _get_crypt_context(cls) -> "CryptContext": """Returns the password encryption context.