From c859465af2c0540cf59110267e415809fa674d5c Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Thu, 18 Apr 2024 09:48:48 +0100 Subject: [PATCH 1/5] refactor(curl-impersonate): Remove manual fix for curl proxy SSL The new version of curl-cffi includes the proper fix for applying ca-bundles to proxy connections making this manual fix no longer required. --- devine/core/downloaders/curl_impersonate.py | 6 ----- poetry.lock | 29 ++++++++++++--------- pyproject.toml | 2 +- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/devine/core/downloaders/curl_impersonate.py b/devine/core/downloaders/curl_impersonate.py index 9f724e3..9245751 100644 --- a/devine/core/downloaders/curl_impersonate.py +++ b/devine/core/downloaders/curl_impersonate.py @@ -6,7 +6,6 @@ from pathlib import Path from typing import Any, Generator, MutableMapping, Optional, Union -from curl_cffi import CurlOpt from curl_cffi.requests import Session from rich import filesize @@ -53,11 +52,6 @@ def download( for one-time request changes like a header, cookie, or proxy. For example, to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`. """ - # https://github.com/yifeikong/curl_cffi/issues/6#issuecomment-2028518677 - # must be applied here since the `session.curl` is thread-localized - # noinspection PyProtectedMember - session.curl.setopt(CurlOpt.PROXY_CAINFO, session.curl._cacert) - save_dir = save_path.parent control_file = save_path.with_name(f"{save_path.name}.!dev") diff --git a/poetry.lock b/poetry.lock index ab7234d..3f8c02f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -523,28 +523,31 @@ testing = ["cssselect", "importlib-resources", "jaraco.test (>=5.1)", "lxml", "p [[package]] name = "curl-cffi" -version = "0.6.2" -description = "libcurl ffi bindings for Python, with impersonation support" +version = "0.7.0b4" +description = "libcurl ffi bindings for Python, with impersonation support." optional = false python-versions = ">=3.8" files = [ - {file = "curl_cffi-0.6.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:23b8a2872b160718c04b06b1f8aa4fb1a2f4f94bce7040493515e081a27cad19"}, - {file = "curl_cffi-0.6.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ad3c1cf5360810825ec4bc3da425f26ee4098878a615dab9d309a99afd883ba9"}, - {file = "curl_cffi-0.6.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d01de6ed737ad1924aaa0198195b9020c38e77ce90ea3d72b9eacf4938c7adf"}, - {file = "curl_cffi-0.6.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37e513cc149d024a2d625e202f2cc9d4423d2937343ea2e06f797d99779e62dc"}, - {file = "curl_cffi-0.6.2-cp38-abi3-win32.whl", hash = "sha256:12e829af97cbf7c1d5afef177e786f6f404ddf163b08897a1ed087cadbeb4837"}, - {file = "curl_cffi-0.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:3791b7a9ae4cb1298165300f2dc2d60a86779f055570ae83163fc2d8a74bf714"}, - {file = "curl_cffi-0.6.2.tar.gz", hash = "sha256:9ee519e960b5fc6e0bbf13d0ecba9ce5f6306cb929354504bf03cc30f59a8f63"}, + {file = "curl_cffi-0.7.0b4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:694d88f7065c59c651970f14bc415431f65ac601a9ba537463d70f432a48ccfc"}, + {file = "curl_cffi-0.7.0b4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6faf01aa8d98d322b877d3d801544692c73729ea6eb4a45af83514a4ecd1c8fe"}, + {file = "curl_cffi-0.7.0b4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d39849371bbf3eab048113693715a8da5c729c494cccfa1128d768d96fdc31e"}, + {file = "curl_cffi-0.7.0b4-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3a5099b98c4bf12cc1afecb3409a9c57e7ebce9447a03c96dfb661ad8fa5e79"}, + {file = "curl_cffi-0.7.0b4-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e3616141a2a0be7896e7dc5da1ed3965e1a78aa2e563d8aba7a641135aeaf1b"}, + {file = "curl_cffi-0.7.0b4-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd16cccc0d3e93c2fbc4f4cb7cce0e10cb2ef7f8957352f3f0d770f0d6e05702"}, + {file = "curl_cffi-0.7.0b4-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:d65aa649abb24020c2ad7b3ce45e2816d1ffe25df06f1a6b0f52fbf353af82e0"}, + {file = "curl_cffi-0.7.0b4-cp38-abi3-win32.whl", hash = "sha256:b55c53bb6dff713cb63f76e2f147e2d54c984b1b09df66b08f52f3acae1aeca0"}, + {file = "curl_cffi-0.7.0b4-cp38-abi3-win_amd64.whl", hash = "sha256:449ab07e07335558997cd62296b5c4f16ce27630de7830e4ad22441049a0ef1e"}, + {file = "curl_cffi-0.7.0b4.tar.gz", hash = "sha256:c09a062b8aac93d4890d2c33b7053c0e1a5cf275328b80c1fb1a950310df75f2"}, ] [package.dependencies] -certifi = "*" +certifi = ">=2024.2.2" cffi = ">=1.12.0" [package.extras] build = ["cibuildwheel", "wheel"] -dev = ["autoflake (==1.4)", "coverage (==6.4.1)", "cryptography (==38.0.3)", "flake8 (==6.0.0)", "flake8-bugbear (==22.7.1)", "flake8-pie (==0.15.0)", "httpx (==0.23.1)", "mypy (==0.971)", "nest-asyncio (==1.6.0)", "pytest (==7.1.2)", "pytest-asyncio (==0.19.0)", "pytest-trio (==0.7.0)", "ruff (==0.1.14)", "trio (==0.21.0)", "trio-typing (==0.7.0)", "trustme (==0.9.0)", "types-certifi (==2021.10.8.2)", "uvicorn (==0.18.3)", "websockets (==11.0.3)"] -test = ["cryptography (==38.0.3)", "fastapi (==0.100.0)", "httpx (==0.23.1)", "nest-asyncio (==1.6.0)", "proxy.py (==2.4.3)", "pytest (==7.1.2)", "pytest-asyncio (==0.19.0)", "pytest-trio (==0.7.0)", "python-multipart (==0.0.6)", "trio (==0.21.0)", "trio-typing (==0.7.0)", "trustme (==0.9.0)", "types-certifi (==2021.10.8.2)", "uvicorn (==0.18.3)", "websockets (==11.0.3)"] +dev = ["charset-normalizer (>=3.3.2,<4.0)", "coverage (>=6.4.1,<7.0)", "cryptography (>=42.0.5,<43.0)", "httpx (==0.23.1)", "mypy (>=1.9.0,<2.0)", "pytest (>=8.1.1,<9.0)", "pytest-asyncio (>=0.23.6,<1.0)", "pytest-trio (>=0.8.0,<1.0)", "ruff (>=0.3.5,<1.0)", "trio (>=0.25.0,<1.0)", "trustme (>=1.1.0,<2.0)", "uvicorn (>=0.29.0,<1.0)", "websockets (>=12.0,<13.0)"] +test = ["charset-normalizer (>=3.3.2,<4.0)", "cryptography (>=42.0.5,<43.0)", "fastapi (==0.110.0)", "httpx (==0.23.1)", "proxy.py (>=2.4.3,<3.0)", "pytest (>=8.1.1,<9.0)", "pytest-asyncio (>=0.23.6,<1.0)", "pytest-trio (>=0.8.0,<1.0)", "python-multipart (>=0.0.9,<1.0)", "trio (>=0.25.0,<1.0)", "trustme (>=1.1.0,<2.0)", "uvicorn (>=0.29.0,<1.0)", "websockets (>=12.0,<13.0)"] [[package]] name = "distlib" @@ -2004,4 +2007,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "db110c0b1b9e30309fcd4e0f80e4369f20b055651f1bef81d0f5e6153a250dec" +content-hash = "4d40927d8e7b25e21de0f56910d6dce05ba2b7c8ce71c465d9e23463f74bc94a" diff --git a/pyproject.toml b/pyproject.toml index b213f81..1e33fe9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ subtitle-filter = "^1.4.9" Unidecode = "^1.3.8" urllib3 = "^2.2.1" chardet = "^5.2.0" -curl-cffi = "^0.6.2" +curl-cffi = "^0.7.0b4" language-data = "^1.2.0" marisa-trie = "^1.1.0" From 4f1dfd7dd1f95dff7238be450c7fc30f5c935465 Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Thu, 18 Apr 2024 09:50:17 +0100 Subject: [PATCH 2/5] refactor(curl-impersonate): Update the default browser to chrome124 --- devine/core/downloaders/curl_impersonate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devine/core/downloaders/curl_impersonate.py b/devine/core/downloaders/curl_impersonate.py index 9245751..bf6e200 100644 --- a/devine/core/downloaders/curl_impersonate.py +++ b/devine/core/downloaders/curl_impersonate.py @@ -17,7 +17,7 @@ RETRY_WAIT = 2 CHUNK_SIZE = 1024 PROGRESS_WINDOW = 5 -BROWSER = config.curl_impersonate.get("browser", "chrome120") +BROWSER = config.curl_impersonate.get("browser", "chrome124") def download( From c101136d554d7adb38c61ae69626998a236caae0 Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Fri, 19 Apr 2024 19:23:56 +0100 Subject: [PATCH 3/5] refactor(Config): Move possible config paths out of func to constant --- devine/core/config.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/devine/core/config.py b/devine/core/config.py index ee9c647..8ca2c57 100644 --- a/devine/core/config.py +++ b/devine/core/config.py @@ -77,29 +77,27 @@ def from_yaml(cls, path: Path) -> Config: return cls(**yaml.safe_load(path.read_text(encoding="utf8")) or {}) -def get_config_path() -> Optional[Path]: - """ - Get Path to Config from various locations. +# noinspection PyProtectedMember +POSSIBLE_CONFIG_PATHS = ( + # The Devine Namespace Folder (e.g., %appdata%/Python/Python311/site-packages/devine) + Config._Directories.namespace_dir / Config._Filenames.root_config, + # The Parent Folder to the Devine Namespace Folder (e.g., %appdata%/Python/Python311/site-packages) + Config._Directories.namespace_dir.parent / Config._Filenames.root_config, + # The AppDirs User Config Folder (e.g., %localappdata%/devine) + Config._Directories.user_configs / Config._Filenames.root_config +) - Looks for a config file in the following folders in order: - 1. The Devine Namespace Folder (e.g., %appdata%/Python/Python311/site-packages/devine) - 2. The Parent Folder to the Devine Namespace Folder (e.g., %appdata%/Python/Python311/site-packages) - 3. The AppDirs User Config Folder (e.g., %localappdata%/devine) +def get_config_path() -> Optional[Path]: + """ + Get Path to Config from any one of the possible locations. Returns None if no config file could be found. """ - # noinspection PyProtectedMember - path = Config._Directories.namespace_dir / Config._Filenames.root_config - if not path.exists(): - # noinspection PyProtectedMember - path = Config._Directories.namespace_dir.parent / Config._Filenames.root_config - if not path.exists(): - # noinspection PyProtectedMember - path = Config._Directories.user_configs / Config._Filenames.root_config - if not path.exists(): - path = None - return path + for path in POSSIBLE_CONFIG_PATHS: + if path.exists(): + return path + return None config_path = get_config_path() From 959b62222e8bcbffe396a68720cc6cc57947a27b Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Fri, 19 Apr 2024 19:26:47 +0100 Subject: [PATCH 4/5] fix(env): List all directories as table in info --- devine/commands/env.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/devine/commands/env.py b/devine/commands/env.py index ce57dbe..0258045 100644 --- a/devine/commands/env.py +++ b/devine/commands/env.py @@ -3,8 +3,11 @@ from typing import Optional import click +from rich.padding import Padding +from rich.table import Table from devine.core.config import config, config_path +from devine.core.console import console from devine.core.constants import context_settings from devine.core.services import Services @@ -18,13 +21,26 @@ def env() -> None: def info() -> None: """Displays information about the current environment.""" log = logging.getLogger("env") - log.info(f"[Config] : {config_path or '--'}") - log.info(f"[Cookies] : {config.directories.cookies}") - log.info(f"[WVDs] : {config.directories.wvds}") - log.info(f"[Cache] : {config.directories.cache}") - log.info(f"[Logs] : {config.directories.logs}") - log.info(f"[Temp Files] : {config.directories.temp}") - log.info(f"[Downloads] : {config.directories.downloads}") + + if config_path: + log.info(f"Config loaded from {config_path}") + else: + log.info("No config file found...") + + table = Table(title="Directories", expand=True) + table.add_column("Name", no_wrap=True) + table.add_column("Path") + + for name in sorted(dir(config.directories)): + if name.startswith("__") or name == "app_dirs": + continue + path = getattr(config.directories, name).resolve() + table.add_row(name.title(), str(path)) + + console.print(Padding( + table, + (1, 5) + )) @env.group(name="clear", short_help="Clear an environment directory.", context_settings=context_settings) From 9768de8bf2b52cfb3f4a17cf0b551c7f752da0dc Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Fri, 19 Apr 2024 19:28:15 +0100 Subject: [PATCH 5/5] feat(env): List possible config path locations when not found --- devine/commands/env.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/devine/commands/env.py b/devine/commands/env.py index 0258045..24ffbac 100644 --- a/devine/commands/env.py +++ b/devine/commands/env.py @@ -5,8 +5,9 @@ import click from rich.padding import Padding from rich.table import Table +from rich.tree import Tree -from devine.core.config import config, config_path +from devine.core.config import POSSIBLE_CONFIG_PATHS, config, config_path from devine.core.console import console from devine.core.constants import context_settings from devine.core.services import Services @@ -25,7 +26,13 @@ def info() -> None: if config_path: log.info(f"Config loaded from {config_path}") else: - log.info("No config file found...") + tree = Tree("No config file found, you can use any of the following locations:") + for i, path in enumerate(POSSIBLE_CONFIG_PATHS, start=1): + tree.add(f"[repr.number]{i}.[/] [text2]{path.resolve()}[/]") + console.print(Padding( + tree, + (0, 5) + )) table = Table(title="Directories", expand=True) table.add_column("Name", no_wrap=True)