From d6a654aaff0a27cddc39084a6a80f874b4def70c Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 10 Oct 2024 12:51:57 +0000 Subject: [PATCH 01/56] feat: Handle `pip install` by `uv` Dirty and untested, but already something working --- .../windmill-worker/src/python_executor.rs | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index df94b7596439a..25a0d8756f1ed 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1015,7 +1015,8 @@ pub async fn handle_python_reqs( "download.config.proto", &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT .replace("{WORKER_DIR}", &worker_dir) - .replace("{CACHE_DIR}", PIP_CACHE_DIR) + // .replace("{CACHE_DIR}", PIP_CACHE_DIR) + .replace("{CACHE_DIR}", UV_CACHE_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; }; @@ -1023,6 +1024,12 @@ pub async fn handle_python_reqs( let mut req_with_penv: Vec<(String, String)> = vec![]; for req in requirements { + // let venv_p = format!( + // "{UV_CACHE_DIR}/{}", + // req.replace(' ', "").replace('/', "").replace(':', "") + // ); + + // tracing::error!("{:?}", &venv_p); let venv_p = format!( "{PIP_CACHE_DIR}/{}", req.replace(' ', "").replace('/', "").replace(':', "") @@ -1147,20 +1154,43 @@ pub async fn handle_python_reqs( let req = format!("{}", req); let mut command_args = vec![ - PYTHON_PATH.as_str(), - "-m", + UV_PATH.as_str(), + // "-m", "pip", "install", &req, - "-I", + // "-I", "--no-deps", "--no-color", - "--isolated", - "--no-warn-conflicts", + // "--isolated", + // Prevent uv from discovering configuration files. + "--no-config", + // "--no-warn-conflicts", "--disable-pip-version-check", - "-t", + // TODO: Doublecheck it + "--system", + // "-t", + "--target", venv_p.as_str(), + // TODO: Use variable instead + "--cache-dir", + UV_CACHE_DIR, ]; + // let mut command_args = vec![ + // PYTHON_PATH.as_str(), + // "-m", + // "pip", + // "install", + // &req, + // "-I", + // "--no-deps", + // "--no-color", + // "--isolated", + // "--no-warn-conflicts", + // "--disable-pip-version-check", + // "-t", + // venv_p.as_str(), + // ]; let pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() .await From 13896a57419c07017bac1e06c399fec36e738a5d Mon Sep 17 00:00:00 2001 From: pyranota Date: Mon, 21 Oct 2024 18:08:35 +0000 Subject: [PATCH 02/56] Integrate with NSJAIL and prepare fallbacks --- backend/windmill-common/src/worker.rs | 1 + .../nsjail/download.py.pip.config.proto | 93 ++++++++++++++ .../nsjail/download_deps.py.pip.sh | 24 ++++ .../nsjail/download_deps.py.sh | 15 ++- .../windmill-worker/src/ansible_executor.rs | 1 + .../windmill-worker/src/python_executor.rs | 114 +++++++++++------- backend/windmill-worker/src/worker.rs | 9 ++ .../windmill-worker/src/worker_lockfiles.rs | 1 + 8 files changed, 211 insertions(+), 47 deletions(-) create mode 100644 backend/windmill-worker/nsjail/download.py.pip.config.proto create mode 100755 backend/windmill-worker/nsjail/download_deps.py.pip.sh diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 37ae61c8d67df..87a8f0fcaf6f7 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -308,6 +308,7 @@ fn parse_file(path: &str) -> Option { pub struct PythonAnnotations { pub no_cache: bool, pub no_uv: bool, + pub no_uv_install: bool, } #[annotations("//")] diff --git a/backend/windmill-worker/nsjail/download.py.pip.config.proto b/backend/windmill-worker/nsjail/download.py.pip.config.proto new file mode 100644 index 0000000000000..15e5c6c7f655d --- /dev/null +++ b/backend/windmill-worker/nsjail/download.py.pip.config.proto @@ -0,0 +1,93 @@ +name: "python download pip" + +mode: ONCE +hostname: "python" +log_level: ERROR +time_limit: 900 + +rlimit_as: 2048 +rlimit_cpu: 1000 +rlimit_fsize: 1024 +rlimit_nofile: 64 + +envar: "HOME=/user" +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +keep_caps: true +keep_env: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + + +mount { + src: "{WORKER_DIR}/download_deps.py.pip.sh" + dst: "/download_deps.sh" + is_bind: true +} + +mount { + src: "{CACHE_DIR}" + dst: "{CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +exec_bin { + path: "/bin/sh" + arg: "/download_deps.sh" +} + diff --git a/backend/windmill-worker/nsjail/download_deps.py.pip.sh b/backend/windmill-worker/nsjail/download_deps.py.pip.sh new file mode 100755 index 0000000000000..efd627483231e --- /dev/null +++ b/backend/windmill-worker/nsjail/download_deps.py.pip.sh @@ -0,0 +1,24 @@ +#/bin/sh + +INDEX_URL_ARG=$([ -z "$INDEX_URL" ] && echo ""|| echo "--index-url $INDEX_URL" ) +EXTRA_INDEX_URL_ARG=$([ -z "$EXTRA_INDEX_URL" ] && echo ""|| echo "--extra-index-url $EXTRA_INDEX_URL" ) +TRUSTED_HOST_ARG=$([ -z "$TRUSTED_HOST" ] && echo "" || echo "--trusted-host $TRUSTED_HOST") + +if [ ! -z "$INDEX_URL" ] +then + echo "\$INDEX_URL is set to $INDEX_URL" +fi + +if [ ! -z "$EXTRA_INDEX_URL" ] +then + echo "\$EXTRA_INDEX_URL is set to $EXTRA_INDEX_URL" +fi + +if [ ! -z "$TRUSTED_HOST" ] +then + echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" +fi + +CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG" +echo $CMD +eval $CMD diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index efd627483231e..3f36f4f7ce2e1 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -19,6 +19,19 @@ then echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" fi -CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG" +CMD="/usr/local/bin/uv pip install +-v \"$REQ\" +--target \"$TARGET\" +--no-cache +--no-config +--no-color +--no-deps +-p 3.11 +--disable-pip-version-check +$INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG +--index-strategy unsafe-best-match +--system +" + echo $CMD eval $CMD diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 9c2474d95f257..deabe07c62d9a 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -116,6 +116,7 @@ async fn handle_ansible_python_deps( job_dir, worker_dir, &mut Some(occupancy_metrics), + false, ) .await?; additional_python_paths.append(&mut venv_path); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index c2b486cd9341b..9e67a79638734 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -42,6 +42,10 @@ lazy_static::lazy_static! { static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") .ok().map(|flag| flag == "true").unwrap_or(false); + /// Use pip install + static ref USE_PIP: bool = std::env::var("USE_PIP") + .ok().map(|flag| flag == "true").unwrap_or(false); + static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); @@ -50,6 +54,8 @@ lazy_static::lazy_static! { } const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto"); +const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK: &str = + include_str!("../nsjail/download.py.pip.config.proto"); const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); @@ -895,10 +901,10 @@ async fn handle_python_deps( .unwrap_or_else(|| vec![]) .clone(); + let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); let requirements = match requirements_o { Some(r) => r, None => { - let annotation = windmill_common::worker::PythonAnnotations::parse(inner_content); let mut already_visited = vec![]; let requirements = windmill_parser_py_imports::parse_python_imports( @@ -923,8 +929,8 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - annotation.no_uv, - annotation.no_cache, + annotations.no_uv, + annotations.no_cache, ) .await .map_err(|e| { @@ -949,6 +955,7 @@ async fn handle_python_deps( job_dir, worker_dir, occupancy_metrics, + annotations.no_uv_install, ) .await?; additional_python_paths.append(&mut venv_path); @@ -960,6 +967,7 @@ lazy_static::lazy_static! { static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap(); } +/// pip install, include cached or pull from S3 pub async fn handle_python_reqs( requirements: Vec<&str>, job_id: &Uuid, @@ -971,12 +979,19 @@ pub async fn handle_python_reqs( job_dir: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + // TODO: Remove (Deprecated) + mut no_uv_install: bool, ) -> error::Result> { let mut req_paths: Vec = vec![]; let mut vars = vec![("PATH", PATH_ENV.as_str())]; let pip_extra_index_url; let pip_index_url; + no_uv_install |= *USE_PIP; + if no_uv_install { + append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await; + tracing::warn!("Fallback to pip"); + } if !*DISABLE_NSJAIL { pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() @@ -1016,11 +1031,14 @@ pub async fn handle_python_reqs( let _ = write_file( job_dir, "download.config.proto", - &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT - .replace("{WORKER_DIR}", &worker_dir) - // .replace("{CACHE_DIR}", PIP_CACHE_DIR) - .replace("{CACHE_DIR}", UV_CACHE_DIR) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + &(if no_uv_install { + NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK + } else { + NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT + }) + .replace("{WORKER_DIR}", &worker_dir) + .replace("{CACHE_DIR}", PIP_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; }; @@ -1156,44 +1174,48 @@ pub async fn handle_python_reqs( #[cfg(windows)] let req = format!("{}", req); - let mut command_args = vec![ - UV_PATH.as_str(), - // "-m", - "pip", - "install", - &req, - // "-I", - "--no-deps", - "--no-color", - // "--isolated", - // Prevent uv from discovering configuration files. - "--no-config", - // "--no-warn-conflicts", - "--disable-pip-version-check", - // TODO: Doublecheck it - "--system", - // "-t", - "--target", - venv_p.as_str(), - // TODO: Use variable instead - "--cache-dir", - UV_CACHE_DIR, - ]; - // let mut command_args = vec![ - // PYTHON_PATH.as_str(), - // "-m", - // "pip", - // "install", - // &req, - // "-I", - // "--no-deps", - // "--no-color", - // "--isolated", - // "--no-warn-conflicts", - // "--disable-pip-version-check", - // "-t", - // venv_p.as_str(), - // ]; + let mut command_args = if no_uv_install { + vec![ + PYTHON_PATH.as_str(), + "-m", + "pip", + "install", + &req, + "-I", + "--no-deps", + "--no-color", + "--isolated", + "--no-warn-conflicts", + "--disable-pip-version-check", + "-t", + venv_p.as_str(), + ] + } else { + vec![ + UV_PATH.as_str(), + "pip", + "install", + &req, + "--no-deps", + "--no-color", + "-p", + "3.11", + // Prevent uv from discovering configuration files. + "--no-config", + // "--no-warn-conflicts", + "--disable-pip-version-check", + // TODO: Doublecheck it + "--system", + // Prefer main index over extra + // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes + // TODO: Use env variable that can be toggled from UI + "--index-strategy", + "unsafe-best-match", + "--target", + venv_p.as_str(), + "--no-cache", + ] + }; let pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() .await diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 83e8adff25a0d..14eab32eac697 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -257,6 +257,7 @@ const NUM_SECS_PING: u64 = 5; const NUM_SECS_READINGS: u64 = 60; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); +const INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK: &str = include_str!("../nsjail/download_deps.py.pip.sh"); pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900; pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days @@ -311,6 +312,7 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + // pub static ref DISABLE_NSJAIL: bool = false; pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL") .ok() .and_then(|x| x.parse::().ok()) @@ -726,6 +728,13 @@ pub async fn run_worker Date: Mon, 21 Oct 2024 18:16:10 +0000 Subject: [PATCH 03/56] Refactor fallback no_uv disable compile and install where no_uv_install and no_uv_compile are a bit more specific --- backend/windmill-worker/src/python_executor.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 9e67a79638734..886d50226a8b9 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -43,7 +43,7 @@ lazy_static::lazy_static! { .ok().map(|flag| flag == "true").unwrap_or(false); /// Use pip install - static ref USE_PIP: bool = std::env::var("USE_PIP") + static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") .ok().map(|flag| flag == "true").unwrap_or(false); @@ -929,7 +929,7 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - annotations.no_uv, + annotations.no_uv || annotations.no_uv_compile, annotations.no_cache, ) .await @@ -955,7 +955,7 @@ async fn handle_python_deps( job_dir, worker_dir, occupancy_metrics, - annotations.no_uv_install, + annotations.no_uv || annotations.no_uv_install, ) .await?; additional_python_paths.append(&mut venv_path); @@ -987,12 +987,15 @@ pub async fn handle_python_reqs( let pip_extra_index_url; let pip_index_url; - no_uv_install |= *USE_PIP; + no_uv_install |= *USE_PIP_INSTALL; + if no_uv_install { append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await; tracing::warn!("Fallback to pip"); } + if !*DISABLE_NSJAIL { + append_logs(&job_id, w_id, "\n Prepare NSJAIL", db).await; pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() .await From bd7607d16201d045c92e7cf32e53aa43ec77700b Mon Sep 17 00:00:00 2001 From: pyranota Date: Mon, 21 Oct 2024 18:23:39 +0000 Subject: [PATCH 04/56] Remove `--disable-pip-version-check` Reason: warning: pip's `--disable-pip-version-check` has no effect --- backend/windmill-worker/nsjail/download_deps.py.sh | 1 - backend/windmill-worker/src/python_executor.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index 3f36f4f7ce2e1..2a63fc482bbad 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -27,7 +27,6 @@ CMD="/usr/local/bin/uv pip install --no-color --no-deps -p 3.11 ---disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG --index-strategy unsafe-best-match --system diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 886d50226a8b9..cabfe394b0505 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1189,7 +1189,6 @@ pub async fn handle_python_reqs( "--no-color", "--isolated", "--no-warn-conflicts", - "--disable-pip-version-check", "-t", venv_p.as_str(), ] From 5805554ac88f848d4239e1173d6623fe5453c3f1 Mon Sep 17 00:00:00 2001 From: pyranota Date: Tue, 22 Oct 2024 14:15:19 +0000 Subject: [PATCH 05/56] Fix backend compilation error --- backend/windmill-common/src/worker.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 87a8f0fcaf6f7..c999770606b4c 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -309,6 +309,7 @@ pub struct PythonAnnotations { pub no_cache: bool, pub no_uv: bool, pub no_uv_install: bool, + pub no_uv_compile: bool, } #[annotations("//")] From a0c86ef4c794aaf25548c93aa7a55f0cd7423ed4 Mon Sep 17 00:00:00 2001 From: pyranota Date: Tue, 22 Oct 2024 14:27:32 +0000 Subject: [PATCH 06/56] Pip fallback overwrite UV's cache --- .../windmill-worker/src/python_executor.rs | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index cabfe394b0505..52bceb235c447 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1059,7 +1059,45 @@ pub async fn handle_python_reqs( req.replace(' ', "").replace('/', "").replace(':', "") ); if metadata(&venv_p).await.is_ok() { - req_paths.push(venv_p); + // TODO: remove (Deperecated) + if no_uv_install { + // e.g.: /tmp/windmill/cache/pip/wmill==1.408.1/wmill-1.408.1.dist-info/INSTALLER + let installer_file_path = format!( + "{PIP_CACHE_DIR}/{}/{}.dist-info/INSTALLER", + req.replace(' ', "").replace('/', "").replace(':', ""), + req.replace(' ', "") + .replace('/', "") + .replace(':', "") + // We want this form of dependency (with _ ) + // typing_extensions-4.12.2.dist-info + .replace('-', "_") + .replace("==", "-") + ); + + append_logs( + &job_id, + w_id, + format!("\nLooking into: {}", installer_file_path), + db, + ) + .await; + + // There is metadata which package manager downloaded library + // It is stored in *.dist-info/INSTALLER + // So if we fallback to pip and we see library installed by uv + // we want to override this installation + // TODO: If error, override anyway + if "uv" == std::fs::read_to_string(installer_file_path)? { + // Rmdir to make it pure + std::fs::remove_dir_all(&venv_p)?; + // Push it for installation + req_with_penv.push((req.to_string(), venv_p)); + } else { + req_paths.push(venv_p); + } + } else { + req_paths.push(venv_p); + } } else { req_with_penv.push((req.to_string(), venv_p)); } From 60e7fa764cb63d78c65ab868b0c08203e907bead Mon Sep 17 00:00:00 2001 From: pyranota Date: Tue, 22 Oct 2024 15:31:56 +0000 Subject: [PATCH 07/56] Initially refactor cache (No S3) --- backend/src/main.rs | 1 - .../windmill-worker/src/python_executor.rs | 65 +++++-------------- backend/windmill-worker/src/worker.rs | 7 ++ 3 files changed, 23 insertions(+), 50 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 84deb4e6b5682..3f04fc86880db 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -892,7 +892,6 @@ pub async fn run_workers = vec![]; for req in requirements { - // let venv_p = format!( - // "{UV_CACHE_DIR}/{}", - // req.replace(' ', "").replace('/', "").replace(':', "") - // ); + let py_prefix = if no_uv_install { + PIP_CACHE_DIR + } else { + PY311_CACHE_DIR + }; - // tracing::error!("{:?}", &venv_p); let venv_p = format!( - "{PIP_CACHE_DIR}/{}", + "{py_prefix}/{}", req.replace(' ', "").replace('/', "").replace(':', "") ); if metadata(&venv_p).await.is_ok() { - // TODO: remove (Deperecated) - if no_uv_install { - // e.g.: /tmp/windmill/cache/pip/wmill==1.408.1/wmill-1.408.1.dist-info/INSTALLER - let installer_file_path = format!( - "{PIP_CACHE_DIR}/{}/{}.dist-info/INSTALLER", - req.replace(' ', "").replace('/', "").replace(':', ""), - req.replace(' ', "") - .replace('/', "") - .replace(':', "") - // We want this form of dependency (with _ ) - // typing_extensions-4.12.2.dist-info - .replace('-', "_") - .replace("==", "-") - ); - - append_logs( - &job_id, - w_id, - format!("\nLooking into: {}", installer_file_path), - db, - ) - .await; - - // There is metadata which package manager downloaded library - // It is stored in *.dist-info/INSTALLER - // So if we fallback to pip and we see library installed by uv - // we want to override this installation - // TODO: If error, override anyway - if "uv" == std::fs::read_to_string(installer_file_path)? { - // Rmdir to make it pure - std::fs::remove_dir_all(&venv_p)?; - // Push it for installation - req_with_penv.push((req.to_string(), venv_p)); - } else { - req_paths.push(venv_p); - } - } else { - req_paths.push(venv_p); - } + req_paths.push(venv_p); } else { req_with_penv.push((req.to_string(), venv_p)); } @@ -1242,8 +1211,6 @@ pub async fn handle_python_reqs( "3.11", // Prevent uv from discovering configuration files. "--no-config", - // "--no-warn-conflicts", - "--disable-pip-version-check", // TODO: Doublecheck it "--system", // Prefer main index over extra diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 14eab32eac697..cbd0cb351fcd2 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -236,7 +236,14 @@ pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs"); pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); +// Used as fallback now pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); + +// pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); +pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); +// pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); +// pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); + pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); From 53909750efc4cb34d2b5304783cde3e837f04f99 Mon Sep 17 00:00:00 2001 From: pyranota Date: Tue, 22 Oct 2024 15:47:47 +0000 Subject: [PATCH 08/56] Support S3 --- backend/windmill-worker/src/global_cache.rs | 26 ++++++++++++++++--- .../windmill-worker/src/python_executor.rs | 7 +++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 12584b905d0bc..57b2b55885ed5 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -20,13 +20,21 @@ use std::sync::Arc; pub async fn build_tar_and_push( s3_client: Arc, folder: String, + no_uv: bool, ) -> error::Result<()> { use object_store::path::Path; + use crate::PY311_CACHE_DIR; + tracing::info!("Started building and pushing piptar {folder}"); let start = Instant::now(); let folder_name = folder.split("/").last().unwrap(); - let tar_path = format!("{PIP_CACHE_DIR}/{folder_name}_tar.tar",); + let prefix = if no_uv { + PIP_CACHE_DIR + } else { + PY311_CACHE_DIR + }; + let tar_path = format!("{prefix}/{folder_name}_tar.tar",); let tar_file = std::fs::File::create(&tar_path)?; let mut tar = tar::Builder::new(tar_file); @@ -46,7 +54,10 @@ pub async fn build_tar_and_push( // })?; if let Err(e) = s3_client .put( - &Path::from(format!("/tar/pip/{folder_name}.tar")), + &Path::from(format!( + "/tar/{}/{folder_name}.tar", + if no_uv { "pip" } else { "python_311" } + )), std::fs::read(&tar_path)?.into(), ) .await @@ -71,7 +82,11 @@ pub async fn build_tar_and_push( } #[cfg(all(feature = "enterprise", feature = "parquet"))] -pub async fn pull_from_tar(client: Arc, folder: String) -> error::Result<()> { +pub async fn pull_from_tar( + client: Arc, + folder: String, + no_uv: bool, +) -> error::Result<()> { use windmill_common::s3_helpers::attempt_fetch_bytes; let folder_name = folder.split("/").last().unwrap(); @@ -79,7 +94,10 @@ pub async fn pull_from_tar(client: Arc, folder: String) -> erro tracing::info!("Attempting to pull piptar {folder_name} from bucket"); let start = Instant::now(); - let tar_path = format!("tar/pip/{folder_name}.tar"); + let tar_path = format!( + "tar/{}/{folder_name}.tar", + if no_uv { "pip" } else { "python_311" } + ); let bytes = attempt_fetch_bytes(client, &tar_path).await?; // tracing::info!("B: {target} {folder}"); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 214c72ebce77f..b8a49c6f7a2f9 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1108,7 +1108,10 @@ pub async fn handle_python_reqs( .map(|(req, venv_p)| { let os = os.clone(); async move { - if pull_from_tar(os, venv_p.clone()).await.is_ok() { + if pull_from_tar(os, venv_p.clone(), no_uv_install) + .await + .is_ok() + { PullFromTar::Pulled(venv_p.to_string()) } else { PullFromTar::NotPulled(req.to_string(), venv_p.to_string()) @@ -1323,7 +1326,7 @@ pub async fn handle_python_reqs( tracing::warn!("S3 cache not available in the pro plan"); } else { let venv_p = venv_p.clone(); - tokio::spawn(build_tar_and_push(os, venv_p)); + tokio::spawn(build_tar_and_push(os, venv_p, no_uv_install)); } } req_paths.push(venv_p); From 414b9c338b79dd64c3ecdb991c4c66e16bfbfdc4 Mon Sep 17 00:00:00 2001 From: pyranota Date: Tue, 22 Oct 2024 16:00:07 +0000 Subject: [PATCH 09/56] Remove unused import --- backend/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 3f04fc86880db..b84de925eff7a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -66,8 +66,8 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, - GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, + GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, POWERSHELL_CACHE_DIR, RUST_CACHE_DIR, + TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ From 1271a65b5a919d2c0fead739032f2629beeb7af5 Mon Sep 17 00:00:00 2001 From: pyranota Date: Tue, 22 Oct 2024 16:00:43 +0000 Subject: [PATCH 10/56] Handle flags for NSJAIL --- .../windmill-worker/src/python_executor.rs | 81 +++++++++++++------ 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b8a49c6f7a2f9..639909c3720ac 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1002,33 +1002,64 @@ pub async fn handle_python_reqs( .clone() .map(handle_ephemeral_token); - if let Some(url) = pip_extra_index_url.as_ref() { - vars.push(("EXTRA_INDEX_URL", url)); - } + if no_uv_install { + if let Some(url) = pip_extra_index_url.as_ref() { + vars.push(("EXTRA_INDEX_URL", url)); + } - pip_index_url = PIP_INDEX_URL - .read() - .await - .clone() - .map(handle_ephemeral_token); + pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); - if let Some(url) = pip_index_url.as_ref() { - vars.push(("INDEX_URL", url)); - } - if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { - vars.push(("PIP_INDEX_CERT", cert_path)); - } - if let Some(host) = PIP_TRUSTED_HOST.as_ref() { - vars.push(("TRUSTED_HOST", host)); - } - if let Some(http_proxy) = HTTP_PROXY.as_ref() { - vars.push(("HTTP_PROXY", http_proxy)); - } - if let Some(https_proxy) = HTTPS_PROXY.as_ref() { - vars.push(("HTTPS_PROXY", https_proxy)); - } - if let Some(no_proxy) = NO_PROXY.as_ref() { - vars.push(("NO_PROXY", no_proxy)); + if let Some(url) = pip_index_url.as_ref() { + vars.push(("INDEX_URL", url)); + } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + vars.push(("PIP_INDEX_CERT", cert_path)); + } + if let Some(host) = PIP_TRUSTED_HOST.as_ref() { + vars.push(("TRUSTED_HOST", host)); + } + if let Some(http_proxy) = HTTP_PROXY.as_ref() { + vars.push(("HTTP_PROXY", http_proxy)); + } + if let Some(https_proxy) = HTTPS_PROXY.as_ref() { + vars.push(("HTTPS_PROXY", https_proxy)); + } + if let Some(no_proxy) = NO_PROXY.as_ref() { + vars.push(("NO_PROXY", no_proxy)); + } + } else { + if let Some(url) = pip_extra_index_url.as_ref() { + vars.push(("UV_EXTRA_INDEX_URL", url)); + } + + pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + + if let Some(url) = pip_index_url.as_ref() { + vars.push(("UV_INDEX_URL", url)); + } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + vars.push(("PIP_INDEX_CERT", cert_path)); + } + if let Some(host) = PIP_TRUSTED_HOST.as_ref() { + vars.push(("TRUSTED_HOST", host)); + } + if let Some(http_proxy) = HTTP_PROXY.as_ref() { + vars.push(("HTTP_PROXY", http_proxy)); + } + if let Some(https_proxy) = HTTPS_PROXY.as_ref() { + vars.push(("HTTPS_PROXY", https_proxy)); + } + if let Some(no_proxy) = NO_PROXY.as_ref() { + vars.push(("NO_PROXY", no_proxy)); + } } let _ = write_file( From 59e8004f994bc0f2aa1689ededd8798bc66623a8 Mon Sep 17 00:00:00 2001 From: pyranota Date: Tue, 22 Oct 2024 16:57:15 +0000 Subject: [PATCH 11/56] Return deleted flag --- backend/windmill-worker/src/python_executor.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 639909c3720ac..7f579abbaad99 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1230,6 +1230,7 @@ pub async fn handle_python_reqs( "--no-color", "--isolated", "--no-warn-conflicts", + "--disable-pip-version-check", "-t", venv_p.as_str(), ] From 3a864c541debdfcee02fa534c6cc87dc6d7efc74 Mon Sep 17 00:00:00 2001 From: pyranota Date: Wed, 23 Oct 2024 11:59:45 +0000 Subject: [PATCH 12/56] Remove verbose mode and enable link-mode=copy --- backend/windmill-worker/nsjail/download_deps.py.sh | 3 ++- backend/windmill-worker/src/python_executor.rs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index 2a63fc482bbad..c17a8ceea81a0 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -20,12 +20,13 @@ then fi CMD="/usr/local/bin/uv pip install --v \"$REQ\" +\"$REQ\" --target \"$TARGET\" --no-cache --no-config --no-color --no-deps +--link-mode=copy -p 3.11 $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG --index-strategy unsafe-best-match diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 7f579abbaad99..e0f8912807588 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1246,6 +1246,7 @@ pub async fn handle_python_reqs( "3.11", // Prevent uv from discovering configuration files. "--no-config", + "--link-mode=copy", // TODO: Doublecheck it "--system", // Prefer main index over extra From 30713a241e1a71b8f1db8e7fb2d45d3317103ee0 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 24 Oct 2024 13:18:48 +0000 Subject: [PATCH 13/56] Granural migration of lockfiles Before i realized we dont need it :) --- backend/windmill-common/src/worker.rs | 29 +++++++ .../windmill-worker/src/ansible_executor.rs | 8 +- .../windmill-worker/src/python_executor.rs | 80 ++++++++++++++++--- .../windmill-worker/src/worker_lockfiles.rs | 17 +++- 4 files changed, 120 insertions(+), 14 deletions(-) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index c999770606b4c..ebbc7a5eac0c9 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -88,6 +88,8 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + pub static ref INSTANCE_PYTHON_VERSION: String = + std::env::var("PYTHON_VERSION").unwrap_or_else(|_| "3.11".to_string()); } pub async fn make_suspended_pull_query(wc: &WorkerConfig) { @@ -310,6 +312,33 @@ pub struct PythonAnnotations { pub no_uv: bool, pub no_uv_install: bool, pub no_uv_compile: bool, + + pub py310: bool, + pub py311: bool, + pub py312: bool, + pub py313: bool, +} + +impl PythonAnnotations { + pub fn get_python_version(&self) -> String { + let mut v: &str = &*INSTANCE_PYTHON_VERSION; + let PythonAnnotations { py310, py311, py312, py313, .. } = *self; + + if py310 { + v = "3.10"; + } + if py311 { + v = "3.11"; + } + if py312 { + v = "3.11"; + } + if py313 { + v = "3.13"; + } + + v.into() + } } #[annotations("//")] diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index deabe07c62d9a..79eee788a71d3 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -21,7 +21,10 @@ use uuid::Uuid; use windmill_common::{ error, jobs::QueuedJob, - worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG}, + worker::{ + to_raw_value, write_file, write_file_at_user_defined_location, PythonAnnotations, + WORKER_CONFIG, + }, }; use windmill_parser_yaml::AnsibleRequirements; use windmill_queue::{append_logs, CanceledBy}; @@ -71,6 +74,7 @@ async fn handle_ansible_python_deps( .unwrap_or_else(|| vec![]) .clone(); + let py_version = PythonAnnotations::default().get_python_version(); let requirements = match requirements_o { Some(r) => r, None => { @@ -90,6 +94,7 @@ async fn handle_ansible_python_deps( worker_name, w_id, &mut Some(occupancy_metrics), + &py_version, false, false, ) @@ -116,6 +121,7 @@ async fn handle_ansible_python_deps( job_dir, worker_dir, &mut Some(occupancy_metrics), + &py_version, false, ) .await?; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index e0f8912807588..8b0bd2dbaca5d 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -16,7 +16,7 @@ use windmill_common::{ error::{self, Error}, jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, utils::calculate_hash, - worker::{write_file, WORKER_CONFIG}, + worker::{write_file, INSTANCE_PYTHON_VERSION, WORKER_CONFIG}, DB, }; @@ -26,6 +26,9 @@ use windmill_common::variables::get_secret_value_as_admin; use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { + + + // Default python static ref PYTHON_PATH: String = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); @@ -50,7 +53,6 @@ lazy_static::lazy_static! { static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); static ref EPHEMERAL_TOKEN_CMD: Option = std::env::var("EPHEMERAL_TOKEN_CMD").ok(); - } const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto"); @@ -108,6 +110,7 @@ pub fn handle_ephemeral_token(x: String) -> String { x } +/// Returns lockfile and python version pub async fn uv_pip_compile( job_id: &Uuid, requirements: &str, @@ -118,6 +121,9 @@ pub async fn uv_pip_compile( worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + // If not set, will default to INSTANCE_PYTHON_VERSION + // If set, then returned python version will be equal + py_version: Option<&str>, // Fallback to pip-compile. Will be removed in future mut no_uv: bool, // Debug-only flag @@ -172,15 +178,70 @@ pub async fn uv_pip_compile( // py-000..000-no_uv } if !no_cache { + /* + There are several scenarious of flow + 1. Cache does not exist for script + 2. Cache exists and in cached lockfile there is no python version + 3. Cache exists and in cached lockfile there is python version + Flows: + 1: We calculate lockfile and add python version to it + 2: Only possible if script exists since versions of windmill that dont support multipython + 1. If INSTANCE_PYTHON_VERSION == 3.11 assign this version to lockfile and return + 3. If INSTANCE_PYTHON_VERSION != 3.11 recalculate lockfile + 3: + 1. if cached_lockfile != annotated_version recalculate lockfile + else: return cache + */ if let Some(cached) = sqlx::query_scalar!( "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + // Python version is not included in hash, + // meaning hash will stay the same independant from python version req_hash ) .fetch_optional(db) .await? { - logs.push_str(&format!("\nfound cached resolution: {req_hash}")); - return Ok(cached); + let cached_version = cached.lines().next().unwrap(); + + if !cached_version.starts_with("# py-") { + // Not possible py_version to be Some + + // if not in form of: # py-3.x + // Assign version to it + // py_version = py_version.or(Some(&*INSTANCE_PYTHON_VERSION)); + + if &*INSTANCE_PYTHON_VERSION == "3.11" { + sqlx::query!( + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + req_hash, + format!("# py-{}\n{cached}",py_version.unwrap_or(&*INSTANCE_PYTHON_VERSION)) + ).fetch_optional(db).await?; + + logs.push_str(&format!( + "\nFound cached resolution without assigned python version: {req_hash}" + )); + logs.push_str(&format!("\nAssigned new python version: 3.11")); + return Ok(cached); + } + // else: + // Recalculate lock if instance python version is not 3.11 + } else if let Some(py_version) = py_version { + // ^^^^ Will be some only if python version explicitly specified by annotation + // Meaning adjusting INSTANCE_PYTHON_VERSION is not affecting existing + if py_version == cached_version { + logs.push_str(&format!("\nfound cached resolution: {req_hash}")); + logs.push_str(&format!("\nAssigned python version: {cached_version}")); + return Ok(cached); + } + } + // Possible only if there is no annotated python version + // And there is python version saved in lockfile + // In that case we dont care what instance python version + else { + logs.push_str(&format!("\nFound cached resolution: {req_hash}")); + logs.push_str(&format!("\nAssigned python version: {cached_version}")); + return Ok(cached); + } } } let file = "requirements.in"; @@ -271,10 +332,8 @@ pub async fn uv_pip_compile( // Target to /tmp/windmill/cache/uv "--cache-dir", UV_CACHE_DIR, - // We dont want UV to manage python installations - "--python-preference", - "only-system", - "--no-python-downloads", + "-p", + py_version, ]; if no_cache { args.extend(["--no-cache"]); @@ -349,7 +408,7 @@ pub async fn uv_pip_compile( sqlx::query!( "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", req_hash, - lockfile + format!("# py-{}\n{lockfile}",py_version.unwrap_or(&*INSTANCE_PYTHON_VERSION)) ).fetch_optional(db).await?; Ok(lockfile) } @@ -929,6 +988,7 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, + &annotations.get_python_version(), annotations.no_uv || annotations.no_uv_compile, annotations.no_cache, ) @@ -955,6 +1015,7 @@ async fn handle_python_deps( job_dir, worker_dir, occupancy_metrics, + &annotations.get_python_version(), annotations.no_uv || annotations.no_uv_install, ) .await?; @@ -979,6 +1040,7 @@ pub async fn handle_python_reqs( job_dir: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + py_version: &str, // TODO: Remove (Deprecated) mut no_uv_install: bool, ) -> error::Result> { diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 44df40232bd57..eb5f3e814a43b 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -12,7 +12,7 @@ use windmill_common::flows::{FlowModule, FlowModuleValue}; use windmill_common::get_latest_deployed_hash_for_path; use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; -use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file}; +use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, PythonAnnotations}; use windmill_common::{ error::{self, to_anyhow}, flows::FlowValue, @@ -1280,8 +1280,10 @@ async fn python_dep( w_id: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + annotations: PythonAnnotations, ) -> std::result::Result { create_dependencies_dir(job_dir).await; + let py_version = annotations.get_python_version(); let req: std::result::Result = uv_pip_compile( job_id, &reqs, @@ -1292,8 +1294,9 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, - false, - false, + &py_version, + annotations.no_uv || annotations.no_uv_compile, + annotations.no_cache, ) .await; // install the dependencies to pre-fill the cache @@ -1309,7 +1312,8 @@ async fn python_dep( job_dir, worker_dir, occupancy_metrics, - false, + &py_version, + annotations.no_uv || annotations.no_uv_install, ) .await; @@ -1343,6 +1347,7 @@ async fn capture_dependency_job( ) -> error::Result { match job_language { ScriptLang::Python3 => { + // panic!("{}", job_raw_code); let reqs = if raw_deps { job_raw_code.to_string() } else { @@ -1359,6 +1364,8 @@ async fn capture_dependency_job( .join("\n") }; + let annotations = windmill_common::worker::PythonAnnotations::parse(job_raw_code); + python_dep( reqs, job_id, @@ -1370,6 +1377,7 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), + annotations, ) .await } @@ -1393,6 +1401,7 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), + PythonAnnotations::default(), ) .await } From b39d25abc2d494eae7d329cbc7428867e1f98071 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 24 Oct 2024 17:31:39 +0000 Subject: [PATCH 14/56] Initial draft (not-working) --- backend/windmill-common/src/worker.rs | 1 + .../windmill-worker/src/ansible_executor.rs | 10 +- .../windmill-worker/src/python_executor.rs | 425 +++++++++++++++--- backend/windmill-worker/src/worker.rs | 12 +- .../windmill-worker/src/worker_lockfiles.rs | 14 +- 5 files changed, 387 insertions(+), 75 deletions(-) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index ebbc7a5eac0c9..4a87cc01b0828 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -306,6 +306,7 @@ fn parse_file(path: &str) -> Option { .flatten() } +#[derive(Copy, Clone)] #[annotations("#")] pub struct PythonAnnotations { pub no_cache: bool, diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 79eee788a71d3..f0aa045d7114d 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -21,10 +21,7 @@ use uuid::Uuid; use windmill_common::{ error, jobs::QueuedJob, - worker::{ - to_raw_value, write_file, write_file_at_user_defined_location, PythonAnnotations, - WORKER_CONFIG, - }, + worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG}, }; use windmill_parser_yaml::AnsibleRequirements; use windmill_queue::{append_logs, CanceledBy}; @@ -74,7 +71,6 @@ async fn handle_ansible_python_deps( .unwrap_or_else(|| vec![]) .clone(); - let py_version = PythonAnnotations::default().get_python_version(); let requirements = match requirements_o { Some(r) => r, None => { @@ -94,7 +90,7 @@ async fn handle_ansible_python_deps( worker_name, w_id, &mut Some(occupancy_metrics), - &py_version, + &mut None, false, false, ) @@ -121,7 +117,7 @@ async fn handle_ansible_python_deps( job_dir, worker_dir, &mut Some(occupancy_metrics), - &py_version, + crate::python_executor::PyVersion::Py311, false, ) .await?; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 8b0bd2dbaca5d..a663582d36246 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1,4 +1,8 @@ -use std::{collections::HashMap, process::Stdio}; +use std::{ + collections::HashMap, + process::Stdio, + sync::{Arc, RwLock}, +}; use itertools::Itertools; use regex::Regex; @@ -16,7 +20,7 @@ use windmill_common::{ error::{self, Error}, jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, utils::calculate_hash, - worker::{write_file, INSTANCE_PYTHON_VERSION, WORKER_CONFIG}, + worker::{write_file, PythonAnnotations, INSTANCE_PYTHON_VERSION, WORKER_CONFIG}, DB, }; @@ -78,6 +82,266 @@ use crate::{ PIP_INDEX_URL, PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, }; +#[derive(Default, Eq, PartialEq, Clone, Copy)] +pub enum PyVersion { + Py310, + // TODO: Default should be inferred from INSTANCE_PYTHON_VERSION + #[default] + Py311, + Py312, + Py313, +} + +impl PyVersion { + /// e.g.: `/tmp/windmill/cache/python_3xy` + pub fn to_cache_dir(&self) -> String { + use windmill_common::worker::ROOT_CACHE_DIR; + format!("{ROOT_CACHE_DIR}python_{}", self.to_string_no_dots()) + } + + /// e.g.: `3xy` + pub fn to_string_no_dots(&self) -> String { + self.to_string_with_dots().replace('.', "") + } + + /// e.g.: `3.xy` + pub fn to_string_with_dots(&self) -> &str { + use PyVersion::*; + match self { + Py310 => "3.10", + Py311 => "3.11", + Py312 => "3.12", + Py313 => "3.13", + } + } + + pub fn from_string_with_dots(value: &str) -> Option { + use PyVersion::*; + match value { + "3.10" => Some(Py310), + "3.11" => Some(Py311), + "3.12" => Some(Py312), + "3.13" => Some(Py313), + _ => None, + } + } + /// e.g.: `# py-3.xy` -> `PyVersion::Py3XY` + pub fn parse_lockfile(line: &str) -> Option { + Self::from_string_with_dots(line.replace("# py-", "").as_str()) + } + + pub fn from_py_annotations(a: PythonAnnotations) -> Option { + let PythonAnnotations { py310, py311, py312, py313, .. } = a; + use PyVersion::*; + if py313 { + Some(Py313) + } else if py312 { + Some(Py312) + } else if py311 { + Some(Py311) + } else if py310 { + Some(Py310) + } else { + None + } + } + + pub async fn install_python( + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + db: &Pool, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + version: &str, + ) -> error::Result<()> { + let logs = String::new(); + // let v_with_dot = self.to_string_with_dots(); + let mut child_cmd = Command::new("uv"); + child_cmd + .current_dir("/tmp/windmill") + .args(["python", "install", version]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let child_process = start_child_process(child_cmd, "uv python install").await?; + + append_logs(&job_id, &w_id, logs, db).await; + handle_child( + job_id, + db, + mem_peak, + &mut None, + child_process, + false, + worker_name, + &w_id, + "uv python install", + None, + false, + occupancy_metrics, + ) + .await + } + + async fn get_python_inner( + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + db: &Pool, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + version: &str, + ) -> error::Result { + let py_path = Self::find_python( + job_id, + mem_peak, + db, + worker_name, + w_id, + occupancy_metrics, + version, + ) + .await; + + // Python is not installed + if py_path.is_err() { + // Install it + Self::install_python( + job_id, + mem_peak, + db, + worker_name, + w_id, + occupancy_metrics, + version, + ) + .await?; + // Try to find one more time + let py_path = Self::find_python( + job_id, + mem_peak, + db, + worker_name, + w_id, + occupancy_metrics, + version, + ) + .await; + + if let Err(ref err) = py_path { + tracing::error!("Cannot find python version {err}"); + } + + // TODO: Cache the result + py_path + } else { + py_path + } + } + + pub async fn get_python( + &self, + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + db: &Pool, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> error::Result { + lazy_static::lazy_static! { + static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); + } + + // TODO + // if let Some(path) = (*PYTHON_PATHS.read().unwrap()).get(self){ + // return Ok(path.as_str()); + // } + + Self::get_python_inner( + job_id, + mem_peak, + db, + worker_name, + w_id, + occupancy_metrics, + self.to_string_with_dots(), + ) + .await + // { + // return Ok(path); + // // PYTHON_PATHS.borrow_mut(). + // } + } + + async fn find_python( + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + db: &Pool, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + version: &str, + // If not set, will default to INSTANCE_PYTHON_VERSION + ) -> error::Result { + let mut logs = String::new(); + // let v_with_dot = self.to_string_with_dots(); + let mut child_cmd = Command::new("uv"); + child_cmd + .current_dir("/tmp/windmill") + .args(["python", "find", version]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let child_process = start_child_process(child_cmd, "uv python find").await?; + + append_logs(&job_id, &w_id, logs, db).await; + handle_child( + job_id, + db, + mem_peak, + &mut None, + child_process, + false, + worker_name, + &w_id, + "uv python find", + None, + false, + occupancy_metrics, + ) + .await; + + Ok("".into()) + // .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + // let output = Command::new("uv") + // .args([ + // "python", + // "find", + // v_with_dot + + // ]) // Add any arguments you want to pass to the command + // .stdout(Stdio::piped()); // Capture the standard output; + // .output() // Execute the command + // .expect("Failed to execute command"); + + // Check if the command was successful + // if output.status.success() { + // // Convert the output to a String and print it + // let stdout = String::from_utf8_lossy(&output.stdout); + // println!("Output:\n{}", stdout); + // } else { + // // If the command failed, print the error + // let stderr = String::from_utf8_lossy(&output.stderr); + // eprintln!("Error:\n{}", stderr); + // } + } +} + #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -122,8 +386,8 @@ pub async fn uv_pip_compile( w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, // If not set, will default to INSTANCE_PYTHON_VERSION - // If set, then returned python version will be equal - py_version: Option<&str>, + // Will always be Some after execution + annotated_py_version: &mut Option, // Fallback to pip-compile. Will be removed in future mut no_uv: bool, // Debug-only flag @@ -132,6 +396,21 @@ pub async fn uv_pip_compile( let mut logs = String::new(); logs.push_str(&format!("\nresolving dependencies...")); logs.push_str(&format!("\ncontent of requirements:\n{}\n", requirements)); + + // Precendence: + // 1. Annotated version + // 2. Instance version + // 3. Hardcoded 3.11 + let mut final_py_version = annotated_py_version.unwrap_or( + PyVersion::from_string_with_dots(&INSTANCE_PYTHON_VERSION).unwrap_or_else(|| { + tracing::error!( + "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to 3.11", + *INSTANCE_PYTHON_VERSION + ); + PyVersion::Py311 + }), + ); + let requirements = if let Some(pip_local_dependencies) = WORKER_CONFIG.read().await.pip_local_dependencies.as_ref() { @@ -201,47 +480,36 @@ pub async fn uv_pip_compile( .fetch_optional(db) .await? { - let cached_version = cached.lines().next().unwrap(); - - if !cached_version.starts_with("# py-") { - // Not possible py_version to be Some - - // if not in form of: # py-3.x - // Assign version to it - // py_version = py_version.or(Some(&*INSTANCE_PYTHON_VERSION)); - - if &*INSTANCE_PYTHON_VERSION == "3.11" { - sqlx::query!( - "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", - req_hash, - format!("# py-{}\n{cached}",py_version.unwrap_or(&*INSTANCE_PYTHON_VERSION)) - ).fetch_optional(db).await?; - - logs.push_str(&format!( - "\nFound cached resolution without assigned python version: {req_hash}" - )); - logs.push_str(&format!("\nAssigned new python version: 3.11")); - return Ok(cached); - } - // else: - // Recalculate lock if instance python version is not 3.11 - } else if let Some(py_version) = py_version { - // ^^^^ Will be some only if python version explicitly specified by annotation - // Meaning adjusting INSTANCE_PYTHON_VERSION is not affecting existing - if py_version == cached_version { - logs.push_str(&format!("\nfound cached resolution: {req_hash}")); - logs.push_str(&format!("\nAssigned python version: {cached_version}")); - return Ok(cached); + if let Some(line) = cached.lines().next() { + if let Some(cached_version) = PyVersion::parse_lockfile(line) { + // If annotated version is given, it should overwrite cached version + if let Some(ref annotated_version) = annotated_py_version { + if *annotated_version == cached_version { + // All good we found cache + logs.push_str(&format!("\nFound cached resolution: {req_hash}")); + return Ok(cached); + } else { + // Annotated version should be used, thus lockfile regenerated + final_py_version = *annotated_version; + } + } else { + } + // But if there is no annotated version provided, then we keep cached version takes + // if cached_version == actual + } else { + tracing::info!( + "There is no assigned python version to script in job: {job_id:?}\n" + ) + // We will assign a python version to this script } + } else { + tracing::error!("No requirement specified in uv_pip_compile"); } - // Possible only if there is no annotated python version - // And there is python version saved in lockfile - // In that case we dont care what instance python version - else { - logs.push_str(&format!("\nFound cached resolution: {req_hash}")); - logs.push_str(&format!("\nAssigned python version: {cached_version}")); - return Ok(cached); - } + + // logs.push_str(&format!("\nFound cached resolution: {req_hash}")); + // // logs.push_str(&format!("\nAssigned python version: {cached_version}")); + // return Ok(cached); + // } } } let file = "requirements.in"; @@ -333,7 +601,7 @@ pub async fn uv_pip_compile( "--cache-dir", UV_CACHE_DIR, "-p", - py_version, + final_py_version.to_string_with_dots(), ]; if no_cache { args.extend(["--no-cache"]); @@ -399,17 +667,26 @@ pub async fn uv_pip_compile( let mut file = File::open(path_lock).await?; let mut req_content = "".to_string(); file.read_to_string(&mut req_content).await?; - let lockfile = req_content - .lines() - .filter(|x| !x.trim_start().starts_with('#')) - .map(|x| x.to_string()) - .collect::>() - .join("\n"); + let lockfile = format!( + "# py-{}\n{}", + final_py_version.to_string_with_dots(), + req_content + .lines() + .filter(|x| !x.trim_start().starts_with('#')) + .map(|x| x.to_string()) + .collect::>() + .join("\n") + ); sqlx::query!( "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", req_hash, - format!("# py-{}\n{lockfile}",py_version.unwrap_or(&*INSTANCE_PYTHON_VERSION)) + // format!("# py-{}\n{lockfile}",py_version.unwrap_or(&*INSTANCE_PYTHON_VERSION)) + // format!("# py-{}\n{lockfile}",py_version) + lockfile ).fetch_optional(db).await?; + + // Give value back + *annotated_py_version = Some(final_py_version); Ok(lockfile) } @@ -432,7 +709,7 @@ pub async fn handle_python_job( occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { let script_path = crate::common::use_flow_root_path(job.script_path()); - let additional_python_paths = handle_python_deps( + let (py_version, additional_python_paths) = handle_python_deps( job_dir, requirements_o, inner_content, @@ -623,6 +900,18 @@ mount {{ "started python code execution {}", job.id ); + + let python_path = py_version + .get_python( + &job.id, + mem_peak, + db, + worker_name, + &job.workspace_id, + &mut Some(occupancy_metrics), + ) + .await?; + let child = if !*DISABLE_NSJAIL { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd @@ -647,7 +936,10 @@ mount {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - let mut python_cmd = Command::new(PYTHON_PATH.as_str()); + let mut python_cmd = Command::new(python_path.as_str()); + let args = vec!["-u", "-m", "wrapper"]; + // let args = vec!["run"]; + // let mut python_cmd = Command::new("uv"); python_cmd .current_dir(job_dir) .env_clear() @@ -657,7 +949,7 @@ mount {{ .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(vec!["-u", "-m", "wrapper"]) + .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -949,7 +1241,7 @@ async fn handle_python_deps( mem_peak: &mut i32, canceled_by: &mut Option, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, -) -> error::Result> { +) -> error::Result<(PyVersion, Vec)> { create_dependencies_dir(job_dir).await; let mut additional_python_paths: Vec = WORKER_CONFIG @@ -961,6 +1253,7 @@ async fn handle_python_deps( .clone(); let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); + let mut version = PyVersion::from_py_annotations(annotations); let requirements = match requirements_o { Some(r) => r, None => { @@ -976,6 +1269,8 @@ async fn handle_python_deps( .await? .join("\n"); if requirements.is_empty() { + // TODO: "# py-3.11".to_string() + // TODO: Still check lockfile "".to_string() } else { uv_pip_compile( @@ -988,7 +1283,7 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - &annotations.get_python_version(), + &mut version, annotations.no_uv || annotations.no_uv_compile, annotations.no_cache, ) @@ -1000,6 +1295,11 @@ async fn handle_python_deps( } }; + let final_version = version.unwrap_or_else(|| { + tracing::error!("Version is supposed to be Some"); + PyVersion::Py311 + }); + if requirements.len() > 0 { let mut venv_path = handle_python_reqs( requirements @@ -1015,13 +1315,13 @@ async fn handle_python_deps( job_dir, worker_dir, occupancy_metrics, - &annotations.get_python_version(), + final_version, annotations.no_uv || annotations.no_uv_install, ) .await?; additional_python_paths.append(&mut venv_path); } - Ok(additional_python_paths) + Ok((final_version, additional_python_paths)) } lazy_static::lazy_static! { @@ -1040,11 +1340,12 @@ pub async fn handle_python_reqs( job_dir: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - py_version: &str, + py_version: PyVersion, // TODO: Remove (Deprecated) mut no_uv_install: bool, ) -> error::Result> { let mut req_paths: Vec = vec![]; + // TODO: Add uv python path and preferences let mut vars = vec![("PATH", PATH_ENV.as_str())]; let pip_extra_index_url; let pip_index_url; @@ -1056,6 +1357,8 @@ pub async fn handle_python_reqs( tracing::warn!("Fallback to pip"); } + // let py_version = requirements[0]; + if !*DISABLE_NSJAIL { append_logs(&job_id, w_id, "\n Prepare NSJAIL", db).await; pip_extra_index_url = PIP_EXTRA_INDEX_URL @@ -1151,7 +1454,7 @@ pub async fn handle_python_reqs( let py_prefix = if no_uv_install { PIP_CACHE_DIR } else { - PY311_CACHE_DIR + &py_version.to_cache_dir() }; let venv_p = format!( @@ -1305,7 +1608,7 @@ pub async fn handle_python_reqs( "--no-deps", "--no-color", "-p", - "3.11", + py_version.to_string_with_dots(), // Prevent uv from discovering configuration files. "--no-config", "--link-mode=copy", diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index cbd0cb351fcd2..83ff6e029f8db 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -239,10 +239,16 @@ pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); // Used as fallback now pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); -// pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); +pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); -// pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); -// pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); +pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); +pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); + +// pub fn PYX_CACHE_DIR(x: &str) -> &str { +// match x { + +// } +// } pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index eb5f3e814a43b..ad96f6eb4384b 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -26,7 +26,9 @@ use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::common::OccupancyMetrics; -use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}; +use crate::python_executor::{ + create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, +}; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; use crate::{ bun_executor::gen_bun_lockfile, @@ -1283,7 +1285,7 @@ async fn python_dep( annotations: PythonAnnotations, ) -> std::result::Result { create_dependencies_dir(job_dir).await; - let py_version = annotations.get_python_version(); + let mut annotated_py_version = PyVersion::from_py_annotations(annotations); let req: std::result::Result = uv_pip_compile( job_id, &reqs, @@ -1294,11 +1296,15 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, - &py_version, + &mut annotated_py_version, annotations.no_uv || annotations.no_uv_compile, annotations.no_cache, ) .await; + let final_version = annotated_py_version.unwrap_or_else(|| { + tracing::error!("Version is supposed to be Some"); + PyVersion::Py311 + }); // install the dependencies to pre-fill the cache if let Ok(req) = req.as_ref() { let r = handle_python_reqs( @@ -1312,7 +1318,7 @@ async fn python_dep( job_dir, worker_dir, occupancy_metrics, - &py_version, + final_version, annotations.no_uv || annotations.no_uv_install, ) .await; From 60f0068ee7f4a89c51a7da50f1244fc7844041ea Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 24 Oct 2024 18:12:42 +0000 Subject: [PATCH 15/56] Add fallback --- backend/windmill-common/src/worker.rs | 1 + .../windmill-worker/src/python_executor.rs | 99 +++++++++++++------ 2 files changed, 71 insertions(+), 29 deletions(-) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 4a87cc01b0828..f062e1378a630 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -313,6 +313,7 @@ pub struct PythonAnnotations { pub no_uv: bool, pub no_uv_install: bool, pub no_uv_compile: bool, + pub no_uv_run: bool, pub py310: bool, pub py311: bool, diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index a663582d36246..b2e9c1ddb6a8f 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -53,6 +53,9 @@ lazy_static::lazy_static! { static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") .ok().map(|flag| flag == "true").unwrap_or(false); + static ref NO_UV_RUN: bool = std::env::var("NO_UV_RUN") + .ok().map(|flag| flag == "true").unwrap_or(false); + static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); @@ -288,35 +291,52 @@ impl PyVersion { version: &str, // If not set, will default to INSTANCE_PYTHON_VERSION ) -> error::Result { - let mut logs = String::new(); + // let mut logs = String::new(); // let v_with_dot = self.to_string_with_dots(); let mut child_cmd = Command::new("uv"); - child_cmd + let output = child_cmd .current_dir("/tmp/windmill") .args(["python", "find", version]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let child_process = start_child_process(child_cmd, "uv python find").await?; + // .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await?; - append_logs(&job_id, &w_id, logs, db).await; - handle_child( - job_id, - db, - mem_peak, - &mut None, - child_process, - false, - worker_name, - &w_id, - "uv python find", - None, - false, - occupancy_metrics, - ) - .await; + // Check if the command was successful + if output.status.success() { + // Convert the output to a String + let stdout = + String::from_utf8(output.stdout).expect("Failed to convert output to String"); + return Ok(stdout); + // println!("Output:\n{}", stdout); + } else { + // If the command failed, print the error + let stderr = + String::from_utf8(output.stderr).expect("Failed to convert error output to String"); + eprintln!("Error:\n{}", stderr); + panic!(); + } - Ok("".into()) + // let child_process = start_child_process(child_cmd, "uv python find").await?; + + // // append_logs(&job_id, &w_id, logs, db).await; + // handle_child( + // job_id, + // db, + // mem_peak, + // &mut None, + // child_process, + // false, + // worker_name, + // &w_id, + // "uv python find", + // None, + // false, + // occupancy_metrics, + // ) + // .await; + + // Ok("".into()) // .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; // let output = Command::new("uv") // .args([ @@ -724,6 +744,9 @@ pub async fn handle_python_job( &mut Some(occupancy_metrics), ) .await?; + let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); + + let no_uv = *NO_UV_RUN | annotations.no_uv | annotations.no_uv_run; append_logs( &job.id, @@ -936,10 +959,15 @@ mount {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - let mut python_cmd = Command::new(python_path.as_str()); + // let mut python_cmd = Command::new(PYTHON_PATH.as_str()); + // tracing::error!("{}", python_path); + let mut python_cmd = if no_uv { + Command::new(PYTHON_PATH.as_str()) + } else { + Command::new(python_path.as_str()) + }; + let args = vec!["-u", "-m", "wrapper"]; - // let args = vec!["run"]; - // let mut python_cmd = Command::new("uv"); python_cmd .current_dir(job_dir) .env_clear() @@ -956,7 +984,11 @@ mount {{ #[cfg(windows)] python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); - start_child_process(python_cmd, PYTHON_PATH.as_str()).await? + if no_uv { + start_child_process(python_cmd, PYTHON_PATH.as_str()).await? + } else { + start_child_process(python_cmd, python_path.as_str()).await? + } }; handle_child( @@ -1357,10 +1389,15 @@ pub async fn handle_python_reqs( tracing::warn!("Fallback to pip"); } - // let py_version = requirements[0]; + if let Err(err) = py_version + .get_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .await + { + tracing::error!("{}", err); + } if !*DISABLE_NSJAIL { - append_logs(&job_id, w_id, "\n Prepare NSJAIL", db).await; + append_logs(&job_id, w_id, "\nPrepare NSJAIL\n", db).await; pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() .await @@ -1451,6 +1488,10 @@ pub async fn handle_python_reqs( let mut req_with_penv: Vec<(String, String)> = vec![]; for req in requirements { + // Ignore # py-3.xy + if req.starts_with('#') { + continue; + } let py_prefix = if no_uv_install { PIP_CACHE_DIR } else { From e62a6a3d6622e7939d3ac1fcee333ccb50260508 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 24 Oct 2024 20:52:48 +0000 Subject: [PATCH 16/56] Fix bug preventing uv from installing deps '\n' - Love it --- .../windmill-worker/src/python_executor.rs | 67 ++++++++++++++----- shell.nix | 2 +- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b2e9c1ddb6a8f..82322aa121b5c 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -150,6 +150,7 @@ impl PyVersion { } pub async fn install_python( + job_dir: &str, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -161,14 +162,14 @@ impl PyVersion { ) -> error::Result<()> { let logs = String::new(); // let v_with_dot = self.to_string_with_dots(); - let mut child_cmd = Command::new("uv"); + let mut child_cmd = Command::new(UV_PATH.as_str()); child_cmd - .current_dir("/tmp/windmill") + .current_dir(job_dir) .args(["python", "install", version]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, "uv python install").await?; + let child_process = start_child_process(child_cmd, "uv").await?; append_logs(&job_id, &w_id, logs, db).await; handle_child( @@ -180,7 +181,7 @@ impl PyVersion { false, worker_name, &w_id, - "uv python install", + "uv", None, false, occupancy_metrics, @@ -189,6 +190,7 @@ impl PyVersion { } async fn get_python_inner( + job_dir: &str, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -199,6 +201,7 @@ impl PyVersion { version: &str, ) -> error::Result { let py_path = Self::find_python( + job_dir, job_id, mem_peak, db, @@ -212,7 +215,8 @@ impl PyVersion { // Python is not installed if py_path.is_err() { // Install it - Self::install_python( + if let Err(err) = Self::install_python( + job_dir, job_id, mem_peak, db, @@ -221,9 +225,13 @@ impl PyVersion { occupancy_metrics, version, ) - .await?; + .await + { + tracing::error!("Cannot install python: {err}"); + } // Try to find one more time let py_path = Self::find_python( + job_dir, job_id, mem_peak, db, @@ -247,6 +255,7 @@ impl PyVersion { pub async fn get_python( &self, + job_dir: &str, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -265,6 +274,7 @@ impl PyVersion { // } Self::get_python_inner( + job_dir, job_id, mem_peak, db, @@ -281,6 +291,7 @@ impl PyVersion { } async fn find_python( + job_dir: &str, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -293,9 +304,9 @@ impl PyVersion { ) -> error::Result { // let mut logs = String::new(); // let v_with_dot = self.to_string_with_dots(); - let mut child_cmd = Command::new("uv"); + let mut child_cmd = Command::new(UV_PATH.as_str()); let output = child_cmd - .current_dir("/tmp/windmill") + .current_dir(job_dir) .args(["python", "find", version]) // .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -307,6 +318,7 @@ impl PyVersion { // Convert the output to a String let stdout = String::from_utf8(output.stdout).expect("Failed to convert output to String"); + tracing::error!("{}", &stdout); return Ok(stdout); // println!("Output:\n{}", stdout); } else { @@ -314,7 +326,9 @@ impl PyVersion { let stderr = String::from_utf8(output.stderr).expect("Failed to convert error output to String"); eprintln!("Error:\n{}", stderr); - panic!(); + tracing::error!("{}", &stderr); + // panic!(); + return Err(error::Error::ExitStatus(1999)); } // let child_process = start_child_process(child_cmd, "uv python find").await?; @@ -926,6 +940,7 @@ mount {{ let python_path = py_version .get_python( + job_dir, &job.id, mem_peak, db, @@ -1389,13 +1404,24 @@ pub async fn handle_python_reqs( tracing::warn!("Fallback to pip"); } - if let Err(err) = py_version - .get_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) - .await - { + let py_path = py_version + .get_python( + job_dir, + job_id, + mem_peak, + db, + worker_name, + w_id, + occupancy_metrics, + ) + .await; + if let Err(ref err) = py_path { tracing::error!("{}", err); } + // TODO: Refactor + let py_path = py_path?.replace('\n', ""); + if !*DISABLE_NSJAIL { append_logs(&job_id, w_id, "\nPrepare NSJAIL\n", db).await; pip_extra_index_url = PIP_EXTRA_INDEX_URL @@ -1649,12 +1675,12 @@ pub async fn handle_python_reqs( "--no-deps", "--no-color", "-p", - py_version.to_string_with_dots(), + &py_path, // Prevent uv from discovering configuration files. "--no-config", "--link-mode=copy", // TODO: Doublecheck it - "--system", + // "--system", // Prefer main index over extra // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes // TODO: Use env variable that can be toggled from UI @@ -1665,6 +1691,7 @@ pub async fn handle_python_reqs( "--no-cache", ] }; + // panic!("{:?}", command_args); let pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() .await @@ -1705,6 +1732,16 @@ pub async fn handle_python_reqs( tracing::debug!("pip install command: {:?}", command_args); + // panic!( + // "{:?}", + // [ + // "-x", + // &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), + // "--command", + // &command_args.join(" "), + // ] + // .join(" ") + // ); #[cfg(unix)] { let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); diff --git a/shell.nix b/shell.nix index c3009eb1830cb..099ca28fed84d 100644 --- a/shell.nix +++ b/shell.nix @@ -27,7 +27,7 @@ in pkgs.mkShell { postgresql watchexec # used in client's dev.nu poetry # for python client - uv + # uv python312Packages.pip-tools # pip-compile ]; From 0fc38cbb9b535f8ee619b36ef294dba21a03e4c2 Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 25 Oct 2024 14:56:15 +0000 Subject: [PATCH 17/56] Add verbosity indicator --- backend/windmill-worker/src/python_executor.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 82322aa121b5c..d3e35cbd58d19 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -765,7 +765,10 @@ pub async fn handle_python_job( append_logs( &job.id, &job.workspace_id, - "\n\n--- PYTHON CODE EXECUTION ---\n".to_string(), + format!( + "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", + py_version.to_string_with_dots() + ), db, ) .await; From c90e6bf17be21098692c248ca7e4fe92277e42d1 Mon Sep 17 00:00:00 2001 From: pyranota Date: Wed, 30 Oct 2024 13:58:43 +0000 Subject: [PATCH 18/56] Iterate on feature - Added instance python version - Rework logic --- backend/src/main.rs | 18 ++- backend/src/monitor.rs | 20 ++- .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/worker.rs | 26 +--- .../windmill-worker/src/ansible_executor.rs | 4 +- .../windmill-worker/src/python_executor.rs | 136 +++++++++++------- backend/windmill-worker/src/worker.rs | 1 + .../windmill-worker/src/worker_lockfiles.rs | 30 +++- .../src/lib/components/instanceSettings.ts | 8 ++ 9 files changed, 149 insertions(+), 95 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index b84de925eff7a..f1d0ee2142ae3 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -9,8 +9,8 @@ use anyhow::Context; use git_version::git_version; use monitor::{ - reload_timeout_wait_result_setting, send_current_log_file_to_object_store, - send_logs_to_object_store, + reload_instance_python_version_setting, reload_timeout_wait_result_setting, + send_current_log_file_to_object_store, send_logs_to_object_store, }; use rand::Rng; use sqlx::{postgres::PgListener, Pool, Postgres}; @@ -34,11 +34,12 @@ use windmill_common::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, - JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, - OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, stats_ee::schedule_stats, @@ -656,6 +657,9 @@ Windmill Community Edition {GIT_VERSION} PIP_INDEX_URL_SETTING => { reload_pip_index_url_setting(&db).await }, + INSTANCE_PYTHON_VERSION_SETTING => { + reload_instance_python_version_setting(&db).await + }, NPM_CONFIG_REGISTRY_SETTING => { reload_npm_config_registry_setting(&db).await }, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 576a38de72a03..b1b69fc3b74a7 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -37,9 +37,9 @@ use windmill_common::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, - KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, - PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, + JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, + OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -60,8 +60,8 @@ use windmill_common::{ use windmill_queue::cancel_job; use windmill_worker::{ create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, - SendResult, BUNFIG_INSTALL_SCOPES, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, - PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY, + SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, + NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY, }; #[cfg(feature = "parquet")] @@ -739,6 +739,16 @@ pub async fn reload_pip_index_url_setting(db: &DB) { .await; } +pub async fn reload_instance_python_version_setting(db: &DB) { + reload_option_setting_with_tracing( + db, + INSTANCE_PYTHON_VERSION_SETTING, + "INSTANCE_PYTHON_VERSION", + INSTANCE_PYTHON_VERSION.clone(), + ) + .await; +} + pub async fn reload_npm_config_registry_setting(db: &DB) { reload_option_setting_with_tracing( db, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 36b80c9fd73fc..5e174f6a4057e 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -12,6 +12,7 @@ pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes"; pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url"; pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url"; +pub const INSTANCE_PYTHON_VERSION_SETTING: &str = "instance_python_version"; pub const SCIM_TOKEN_SETTING: &str = "scim_token"; pub const SAML_METADATA_SETTING: &str = "saml_metadata"; pub const SMTP_SETTING: &str = "smtp_settings"; diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index f062e1378a630..562adb9ff5bcc 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -88,8 +88,8 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); - pub static ref INSTANCE_PYTHON_VERSION: String = - std::env::var("PYTHON_VERSION").unwrap_or_else(|_| "3.11".to_string()); + // pub static ref INSTANCE_PYTHON_VERSION: String = + // std::env::var("PYTHON_VERSION").unwrap_or_else(|_| "3.11".to_string()); } pub async fn make_suspended_pull_query(wc: &WorkerConfig) { @@ -321,28 +321,6 @@ pub struct PythonAnnotations { pub py313: bool, } -impl PythonAnnotations { - pub fn get_python_version(&self) -> String { - let mut v: &str = &*INSTANCE_PYTHON_VERSION; - let PythonAnnotations { py310, py311, py312, py313, .. } = *self; - - if py310 { - v = "3.10"; - } - if py311 { - v = "3.11"; - } - if py312 { - v = "3.11"; - } - if py313 { - v = "3.13"; - } - - v.into() - } -} - #[annotations("//")] pub struct TypeScriptAnnotations { pub npm: bool, diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index f0aa045d7114d..0bcc815bc3d49 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -33,7 +33,7 @@ use crate::{ OccupancyMetrics, }, handle_child::handle_child, - python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, + python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV, }; @@ -90,7 +90,7 @@ async fn handle_ansible_python_deps( worker_name, w_id, &mut Some(occupancy_metrics), - &mut None, + PyVersion::Py311, false, false, ) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index d3e35cbd58d19..ba7fd4259649b 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -5,6 +5,7 @@ use std::{ }; use itertools::Itertools; +use nix::NixPath; use regex::Regex; use serde_json::value::RawValue; use sqlx::{types::Json, Pool, Postgres}; @@ -20,7 +21,7 @@ use windmill_common::{ error::{self, Error}, jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, utils::calculate_hash, - worker::{write_file, PythonAnnotations, INSTANCE_PYTHON_VERSION, WORKER_CONFIG}, + worker::{write_file, PythonAnnotations, WORKER_CONFIG}, DB, }; @@ -81,8 +82,8 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, - LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, + INSTANCE_PYTHON_VERSION, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, + PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, }; #[derive(Default, Eq, PartialEq, Clone, Copy)] @@ -421,7 +422,7 @@ pub async fn uv_pip_compile( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, // If not set, will default to INSTANCE_PYTHON_VERSION // Will always be Some after execution - annotated_py_version: &mut Option, + py_version: PyVersion, // Fallback to pip-compile. Will be removed in future mut no_uv: bool, // Debug-only flag @@ -431,19 +432,7 @@ pub async fn uv_pip_compile( logs.push_str(&format!("\nresolving dependencies...")); logs.push_str(&format!("\ncontent of requirements:\n{}\n", requirements)); - // Precendence: - // 1. Annotated version - // 2. Instance version - // 3. Hardcoded 3.11 - let mut final_py_version = annotated_py_version.unwrap_or( - PyVersion::from_string_with_dots(&INSTANCE_PYTHON_VERSION).unwrap_or_else(|| { - tracing::error!( - "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to 3.11", - *INSTANCE_PYTHON_VERSION - ); - PyVersion::Py311 - }), - ); + // New version, the version what we wanna have let requirements = if let Some(pip_local_dependencies) = WORKER_CONFIG.read().await.pip_local_dependencies.as_ref() @@ -516,34 +505,38 @@ pub async fn uv_pip_compile( { if let Some(line) = cached.lines().next() { if let Some(cached_version) = PyVersion::parse_lockfile(line) { - // If annotated version is given, it should overwrite cached version - if let Some(ref annotated_version) = annotated_py_version { - if *annotated_version == cached_version { - // All good we found cache - logs.push_str(&format!("\nFound cached resolution: {req_hash}")); - return Ok(cached); - } else { - // Annotated version should be used, thus lockfile regenerated - final_py_version = *annotated_version; - } - } else { + // We should overwrite any version in lockfile + // And only return cache if cached_version == new_version + // This will trigger recompilation for all deps, + // but it is ok, since we do only on deploy and cached lockfiles for non-deployed scripts + // are being cleaned up every 3 days anyway + if py_version == cached_version { + // All good we found cache + logs.push_str(&format!("\nFound cached resolution: {req_hash}")); + return Ok(cached); } - // But if there is no annotated version provided, then we keep cached version takes - // if cached_version == actual + // Annotated version should be used, thus lockfile regenerated } else { tracing::info!( "There is no assigned python version to script in job: {job_id:?}\n" - ) + ); // We will assign a python version to this script } + // TODO: Small optimisation + // We end up here only if we try to redeploy script without assigned version + // So we could check if final_version == 3.11 and if so, assign python version (write to lockfile) + // and return cache + // else if new_py_version == PyVersion::Py311 { + // tracing::info!( + // "There is no assigned python version to script in job: {job_id:?}\n" + // ); + // logs.push_str(&format!("\nFound cached resolution: {req_hash}")); + // return Ok(cached); + // // We will assign a python version to this script + // } } else { tracing::error!("No requirement specified in uv_pip_compile"); } - - // logs.push_str(&format!("\nFound cached resolution: {req_hash}")); - // // logs.push_str(&format!("\nAssigned python version: {cached_version}")); - // return Ok(cached); - // } } } let file = "requirements.in"; @@ -635,7 +628,7 @@ pub async fn uv_pip_compile( "--cache-dir", UV_CACHE_DIR, "-p", - final_py_version.to_string_with_dots(), + py_version.to_string_with_dots(), ]; if no_cache { args.extend(["--no-cache"]); @@ -703,7 +696,7 @@ pub async fn uv_pip_compile( file.read_to_string(&mut req_content).await?; let lockfile = format!( "# py-{}\n{}", - final_py_version.to_string_with_dots(), + py_version.to_string_with_dots(), req_content .lines() .filter(|x| !x.trim_start().starts_with('#')) @@ -719,8 +712,6 @@ pub async fn uv_pip_compile( lockfile ).fetch_optional(db).await?; - // Give value back - *annotated_py_version = Some(final_py_version); Ok(lockfile) } @@ -951,7 +942,8 @@ mount {{ &job.workspace_id, &mut Some(occupancy_metrics), ) - .await?; + .await? + .replace('\n', ""); let child = if !*DISABLE_NSJAIL { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -1303,7 +1295,27 @@ async fn handle_python_deps( .clone(); let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); - let mut version = PyVersion::from_py_annotations(annotations); + // let mut version = PyVersion::from_py_annotations(annotations) + // .or(PyVersion::from_string_with_dots(&*INSTANCE_PYTHON_VERSION)); + // let mut version = PyVersion::from_py_annotations(annotations) + // .or(PyVersion::from_string_with_dots(&*INSTANCE_PYTHON_VERSION)); + // Precendence: + // 1. Annotated version + // 2. Instance version + // 3. Hardcoded 3.11 + + let instance_version = + (INSTANCE_PYTHON_VERSION.read().await.clone()).unwrap_or("3.11".to_owned()); + + let annotated_or_default_version = PyVersion::from_py_annotations(annotations).unwrap_or( + PyVersion::from_string_with_dots(&instance_version).unwrap_or_else(|| { + tracing::error!( + "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to 3.11", + *INSTANCE_PYTHON_VERSION + ); + PyVersion::Py311 + }), + ); let requirements = match requirements_o { Some(r) => r, None => { @@ -1333,7 +1345,7 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - &mut version, + annotated_or_default_version, annotations.no_uv || annotations.no_uv_compile, annotations.no_cache, ) @@ -1345,17 +1357,39 @@ async fn handle_python_deps( } }; - let final_version = version.unwrap_or_else(|| { - tracing::error!("Version is supposed to be Some"); - PyVersion::Py311 - }); + // let final_version = version.unwrap_or_else(|| { + // tracing::error!("Version is supposed to be Some"); + // PyVersion::Py311 + // }); + + // Read more in next comment section + let mut final_version = PyVersion::Py311; if requirements.len() > 0 { + let req: Vec<&str> = requirements + .split("\n") + .filter(|x| !x.starts_with("--")) + .collect(); + + // uv_pip_compile stage will be skipped for deployed scripts. + // Leaving us with 2 scenarious: + // 1. We have version in lockfile + // 2. We dont + // + // We want to use 3.11 version for scripts without assigned python version + // Because this means that this script was deployed before multiple python version support was introduced + // And the default version of python before this point was 3.11 + // + // But for 1. we just parse line to get version + + // Parse lockfile's line and if there is no version, fallback to annotation_default + if let Some(v) = PyVersion::parse_lockfile(&req[0]) { + final_version = v; + } + // final_version = PyVersion::parse_lockfile(&req[0]).unwrap_or(final_version); + let mut venv_path = handle_python_reqs( - requirements - .split("\n") - .filter(|x| !x.starts_with("--")) - .collect(), + req, job_id, w_id, mem_peak, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 83ff6e029f8db..089e349fa2839 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -362,6 +362,7 @@ lazy_static::lazy_static! { pub static ref PIP_EXTRA_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); + pub static ref INSTANCE_PYTHON_VERSION: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index ad96f6eb4384b..91c0d7a692fc7 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -30,6 +30,7 @@ use crate::python_executor::{ create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, }; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; +use crate::INSTANCE_PYTHON_VERSION; use crate::{ bun_executor::gen_bun_lockfile, deno_executor::generate_deno_lock, @@ -1285,7 +1286,20 @@ async fn python_dep( annotations: PythonAnnotations, ) -> std::result::Result { create_dependencies_dir(job_dir).await; - let mut annotated_py_version = PyVersion::from_py_annotations(annotations); + // let mut annotated_py_version = PyVersion::from_py_annotations(annotations); + + let instance_version = + (INSTANCE_PYTHON_VERSION.read().await.clone()).unwrap_or("3.11".to_owned()); + let final_version = PyVersion::from_py_annotations(annotations).unwrap_or( + PyVersion::from_string_with_dots(&instance_version).unwrap_or_else(|| { + tracing::error!( + "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to 3.11", + *INSTANCE_PYTHON_VERSION + ); + PyVersion::Py311 + }), + ); + let req: std::result::Result = uv_pip_compile( job_id, &reqs, @@ -1296,15 +1310,15 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, - &mut annotated_py_version, + final_version, annotations.no_uv || annotations.no_uv_compile, annotations.no_cache, ) .await; - let final_version = annotated_py_version.unwrap_or_else(|| { - tracing::error!("Version is supposed to be Some"); - PyVersion::Py311 - }); + // let final_version = annotated_py_version.unwrap_or_else(|| { + // tracing::error!("Version is supposed to be Some"); + // PyVersion::Py311 + // }); // install the dependencies to pre-fill the cache if let Ok(req) = req.as_ref() { let r = handle_python_reqs( @@ -1318,6 +1332,10 @@ async fn python_dep( job_dir, worker_dir, occupancy_metrics, + // In this case we calculate lockfile each time and it is guranteed + // that version in lockfile will be equal to final_version + // So we can skip parsing of lockfile to get python version + // and instead just use final_version final_version, annotations.no_uv || annotations.no_uv_install, ) diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 4bb35371dc5a1..4d69ac90b76ad 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -168,6 +168,14 @@ export const settings: Record = { storage: 'setting', ee_only: '' }, + { + label: 'Instance Python Version', + description: 'Default python version for newly deployed scripts', + key: 'instance_python_version', + fieldType: 'text', + placeholder: '3.11', + storage: 'setting', + }, { label: 'Npm Config Registry', description: 'Add private npm registry', From 01b9a352e71c31aaa3a4bf33f4f582abfbc14a6c Mon Sep 17 00:00:00 2001 From: pyranota Date: Wed, 30 Oct 2024 14:19:01 +0000 Subject: [PATCH 19/56] Fix EE build error error[E0599]: no method named `iter` found for tuple `(PyVersion, std::vec::Vec)` in the current scope --- backend/windmill-worker/src/python_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index ba7fd4259649b..6086789dc0d21 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1895,7 +1895,7 @@ pub async fn start_worker( .to_vec(); let context_envs = build_envs_map(context).await; - let additional_python_paths = handle_python_deps( + let (_, additional_python_paths) = handle_python_deps( job_dir, requirements_o, inner_content, From 2c41219f83070a3019dfd7d1a990627ba63964ee Mon Sep 17 00:00:00 2001 From: pyranota Date: Wed, 30 Oct 2024 14:54:38 +0000 Subject: [PATCH 20/56] Support S3 --- backend/windmill-worker/src/global_cache.rs | 26 ++---- .../windmill-worker/src/python_executor.rs | 93 ++++++------------- 2 files changed, 34 insertions(+), 85 deletions(-) diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 57b2b55885ed5..3c8b27d303206 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -20,21 +20,17 @@ use std::sync::Arc; pub async fn build_tar_and_push( s3_client: Arc, folder: String, - no_uv: bool, + // /tmp/windmill/cache/python_311 + cache_dir: String, + // python_311 + prefix: String, ) -> error::Result<()> { use object_store::path::Path; - use crate::PY311_CACHE_DIR; - tracing::info!("Started building and pushing piptar {folder}"); let start = Instant::now(); let folder_name = folder.split("/").last().unwrap(); - let prefix = if no_uv { - PIP_CACHE_DIR - } else { - PY311_CACHE_DIR - }; - let tar_path = format!("{prefix}/{folder_name}_tar.tar",); + let tar_path = format!("{cache_dir}/{folder_name}_tar.tar",); let tar_file = std::fs::File::create(&tar_path)?; let mut tar = tar::Builder::new(tar_file); @@ -54,10 +50,7 @@ pub async fn build_tar_and_push( // })?; if let Err(e) = s3_client .put( - &Path::from(format!( - "/tar/{}/{folder_name}.tar", - if no_uv { "pip" } else { "python_311" } - )), + &Path::from(format!("/tar/{prefix}/{folder_name}.tar")), std::fs::read(&tar_path)?.into(), ) .await @@ -85,7 +78,7 @@ pub async fn build_tar_and_push( pub async fn pull_from_tar( client: Arc, folder: String, - no_uv: bool, + prefix: String, ) -> error::Result<()> { use windmill_common::s3_helpers::attempt_fetch_bytes; @@ -94,10 +87,7 @@ pub async fn pull_from_tar( tracing::info!("Attempting to pull piptar {folder_name} from bucket"); let start = Instant::now(); - let tar_path = format!( - "tar/{}/{folder_name}.tar", - if no_uv { "pip" } else { "python_311" } - ); + let tar_path = format!("tar/{prefix}/{folder_name}.tar"); let bytes = attempt_fetch_bytes(client, &tar_path).await?; // tracing::info!("B: {target} {folder}"); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 6086789dc0d21..15e912dcca6af 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -86,10 +86,10 @@ use crate::{ PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, }; +// TODO: Default should be removed #[derive(Default, Eq, PartialEq, Clone, Copy)] pub enum PyVersion { Py310, - // TODO: Default should be inferred from INSTANCE_PYTHON_VERSION #[default] Py311, Py312, @@ -100,14 +100,21 @@ impl PyVersion { /// e.g.: `/tmp/windmill/cache/python_3xy` pub fn to_cache_dir(&self) -> String { use windmill_common::worker::ROOT_CACHE_DIR; - format!("{ROOT_CACHE_DIR}python_{}", self.to_string_no_dots()) + format!("{ROOT_CACHE_DIR}python_{}", &self.to_cache_dir_top_level()) + } + /// e.g.: `python_3xy` + pub fn to_cache_dir_top_level(&self) -> String { + format!("python_{}", self.to_string_no_dots()) + } + /// e.g.: `(to_cache_dir(), to_cache_dir_top_level())` + pub fn to_cache_dir_tuple(&self) -> (String, String) { + let top_level = self.to_cache_dir_top_level(); + (format!("{ROOT_CACHE_DIR}python_{}", &top_level), top_level) } - /// e.g.: `3xy` pub fn to_string_no_dots(&self) -> String { self.to_string_with_dots().replace('.', "") } - /// e.g.: `3.xy` pub fn to_string_with_dots(&self) -> &str { use PyVersion::*; @@ -118,7 +125,6 @@ impl PyVersion { Py313 => "3.13", } } - pub fn from_string_with_dots(value: &str) -> Option { use PyVersion::*; match value { @@ -133,7 +139,6 @@ impl PyVersion { pub fn parse_lockfile(line: &str) -> Option { Self::from_string_with_dots(line.replace("# py-", "").as_str()) } - pub fn from_py_annotations(a: PythonAnnotations) -> Option { let PythonAnnotations { py310, py311, py312, py313, .. } = a; use PyVersion::*; @@ -149,7 +154,6 @@ impl PyVersion { None } } - pub async fn install_python( job_dir: &str, job_id: &Uuid, @@ -189,7 +193,6 @@ impl PyVersion { ) .await } - async fn get_python_inner( job_dir: &str, job_id: &Uuid, @@ -253,7 +256,6 @@ impl PyVersion { py_path } } - pub async fn get_python( &self, job_dir: &str, @@ -269,11 +271,6 @@ impl PyVersion { static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); } - // TODO - // if let Some(path) = (*PYTHON_PATHS.read().unwrap()).get(self){ - // return Ok(path.as_str()); - // } - Self::get_python_inner( job_dir, job_id, @@ -285,12 +282,7 @@ impl PyVersion { self.to_string_with_dots(), ) .await - // { - // return Ok(path); - // // PYTHON_PATHS.borrow_mut(). - // } } - async fn find_python( job_dir: &str, job_id: &Uuid, @@ -331,49 +323,6 @@ impl PyVersion { // panic!(); return Err(error::Error::ExitStatus(1999)); } - - // let child_process = start_child_process(child_cmd, "uv python find").await?; - - // // append_logs(&job_id, &w_id, logs, db).await; - // handle_child( - // job_id, - // db, - // mem_peak, - // &mut None, - // child_process, - // false, - // worker_name, - // &w_id, - // "uv python find", - // None, - // false, - // occupancy_metrics, - // ) - // .await; - - // Ok("".into()) - // .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; - // let output = Command::new("uv") - // .args([ - // "python", - // "find", - // v_with_dot - - // ]) // Add any arguments you want to pass to the command - // .stdout(Stdio::piped()); // Capture the standard output; - // .output() // Execute the command - // .expect("Failed to execute command"); - - // Check if the command was successful - // if output.status.success() { - // // Convert the output to a String and print it - // let stdout = String::from_utf8_lossy(&output.stdout); - // println!("Output:\n{}", stdout); - // } else { - // // If the command failed, print the error - // let stderr = String::from_utf8_lossy(&output.stderr); - // eprintln!("Error:\n{}", stderr); - // } } } @@ -1602,16 +1551,19 @@ pub async fn handle_python_reqs( }); let start = std::time::Instant::now(); + let prefix = if no_uv { + "pip".to_owned() + } else { + py_version.to_cache_dir_top_level() + }; + let futures = req_with_penv .clone() .into_iter() .map(|(req, venv_p)| { let os = os.clone(); async move { - if pull_from_tar(os, venv_p.clone(), no_uv_install) - .await - .is_ok() - { + if pull_from_tar(os, venv_p.clone(), prefix).await.is_ok() { PullFromTar::Pulled(venv_p.to_string()) } else { PullFromTar::NotPulled(req.to_string(), venv_p.to_string()) @@ -1839,7 +1791,14 @@ pub async fn handle_python_reqs( tracing::warn!("S3 cache not available in the pro plan"); } else { let venv_p = venv_p.clone(); - tokio::spawn(build_tar_and_push(os, venv_p, no_uv_install)); + + let (cache_dir, prefix) = if no_uv { + (PIP_CACHE_DIR.to_owned(), "pip".to_owned()) + } else { + py_version.to_cache_dir_tuple() + }; + + tokio::spawn(build_tar_and_push(os, venv_p, cache_dir, prefix)); } } req_paths.push(venv_p); From cfd45cadd642a3c4c10d34bc7027235e13ad500a Mon Sep 17 00:00:00 2001 From: pyranota Date: Wed, 30 Oct 2024 15:29:05 +0000 Subject: [PATCH 21/56] Support NSJAIL --- .../windmill-worker/nsjail/download_deps.py.sh | 2 +- backend/windmill-worker/src/python_executor.rs | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index c17a8ceea81a0..afed8bf08d6c4 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -27,7 +27,7 @@ CMD="/usr/local/bin/uv pip install --no-color --no-deps --link-mode=copy --p 3.11 +-p $PY_PATH $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG --index-strategy unsafe-best-match --system diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 15e912dcca6af..29010a9e12e5a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -171,6 +171,11 @@ impl PyVersion { child_cmd .current_dir(job_dir) .args(["python", "install", version]) + // TODO: Do we need these? + .envs([ + ("UV_PYTHON_INSTALL_DIR", "/tmp/windmill/cache/python"), + ("UV_PYTHON_PREFERENCE", "only-managed"), + ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -301,6 +306,10 @@ impl PyVersion { let output = child_cmd .current_dir(job_dir) .args(["python", "find", version]) + .envs([ + ("UV_PYTHON_INSTALL_DIR", "/tmp/windmill/cache/python"), + ("UV_PYTHON_PREFERENCE", "only-managed"), + ]) // .stdout(Stdio::piped()) .stderr(Stdio::piped()) .output() @@ -909,7 +918,11 @@ mount {{ "--config", "run.config.proto", "--", - PYTHON_PATH.as_str(), + if no_uv { + PYTHON_PATH.as_str() + } else { + &python_path + }, "-u", "-m", "wrapper", @@ -1622,6 +1635,9 @@ pub async fn handle_python_reqs( let req = req.to_string(); vars.push(("REQ", &req)); vars.push(("TARGET", &venv_p)); + if !no_uv { + vars.push(("PY_PATH", &py_path)); + } let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) From c8796249eeaaf4f093acc5ccca52bcf7a07d7131 Mon Sep 17 00:00:00 2001 From: pyranota Date: Wed, 30 Oct 2024 16:27:08 +0000 Subject: [PATCH 22/56] Refactor `get_python` --- backend/windmill-common/src/error.rs | 2 + backend/windmill-worker/src/global_cache.rs | 3 - .../windmill-worker/src/python_executor.rs | 76 +++++-------------- 3 files changed, 23 insertions(+), 58 deletions(-) diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 8ac8537c11c6c..f0fe25f674efa 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -62,6 +62,8 @@ pub enum Error { OpenAIError(String), #[error("{0}")] AlreadyCompleted(String), + #[error("Find python error: {0}")] + FindPythonError(String), } impl Error { diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 3c8b27d303206..ef367b41eadce 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -1,6 +1,3 @@ -#[cfg(all(feature = "enterprise", feature = "parquet"))] -use crate::PIP_CACHE_DIR; - // #[cfg(feature = "enterprise")] // use rand::Rng; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 29010a9e12e5a..65ee631e17b61 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -5,7 +5,6 @@ use std::{ }; use itertools::Itertools; -use nix::NixPath; use regex::Regex; use serde_json::value::RawValue; use sqlx::{types::Json, Pool, Postgres}; @@ -86,11 +85,9 @@ use crate::{ PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, }; -// TODO: Default should be removed -#[derive(Default, Eq, PartialEq, Clone, Copy)] +#[derive(Eq, PartialEq, Clone, Copy)] pub enum PyVersion { Py310, - #[default] Py311, Py312, Py313, @@ -108,6 +105,7 @@ impl PyVersion { } /// e.g.: `(to_cache_dir(), to_cache_dir_top_level())` pub fn to_cache_dir_tuple(&self) -> (String, String) { + use windmill_common::worker::ROOT_CACHE_DIR; let top_level = self.to_cache_dir_top_level(); (format!("{ROOT_CACHE_DIR}python_{}", &top_level), top_level) } @@ -209,17 +207,7 @@ impl PyVersion { occupancy_metrics: &mut Option<&mut OccupancyMetrics>, version: &str, ) -> error::Result { - let py_path = Self::find_python( - job_dir, - job_id, - mem_peak, - db, - worker_name, - w_id, - occupancy_metrics, - version, - ) - .await; + let py_path = Self::find_python(job_dir, version).await; // Python is not installed if py_path.is_err() { @@ -237,26 +225,19 @@ impl PyVersion { .await { tracing::error!("Cannot install python: {err}"); - } - // Try to find one more time - let py_path = Self::find_python( - job_dir, - job_id, - mem_peak, - db, - worker_name, - w_id, - occupancy_metrics, - version, - ) - .await; + return Err(err); + } else { + // Try to find one more time + let py_path = Self::find_python(job_dir, version).await; - if let Err(ref err) = py_path { - tracing::error!("Cannot find python version {err}"); - } + if let Err(err) = py_path { + tracing::error!("Cannot find python version {err}"); + return Err(err); + } - // TODO: Cache the result - py_path + // TODO: Cache the result + py_path + } } else { py_path } @@ -288,18 +269,7 @@ impl PyVersion { ) .await } - async fn find_python( - job_dir: &str, - job_id: &Uuid, - mem_peak: &mut i32, - // canceled_by: &mut Option, - db: &Pool, - worker_name: &str, - w_id: &str, - occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - version: &str, - // If not set, will default to INSTANCE_PYTHON_VERSION - ) -> error::Result { + async fn find_python(job_dir: &str, version: &str) -> error::Result { // let mut logs = String::new(); // let v_with_dot = self.to_string_with_dots(); let mut child_cmd = Command::new(UV_PATH.as_str()); @@ -320,17 +290,12 @@ impl PyVersion { // Convert the output to a String let stdout = String::from_utf8(output.stdout).expect("Failed to convert output to String"); - tracing::error!("{}", &stdout); - return Ok(stdout); - // println!("Output:\n{}", stdout); + return Ok(stdout.replace('\n', "")); } else { // If the command failed, print the error let stderr = String::from_utf8(output.stderr).expect("Failed to convert error output to String"); - eprintln!("Error:\n{}", stderr); - tracing::error!("{}", &stderr); - // panic!(); - return Err(error::Error::ExitStatus(1999)); + return Err(error::Error::FindPythonError(stderr)); } } } @@ -1564,7 +1529,7 @@ pub async fn handle_python_reqs( }); let start = std::time::Instant::now(); - let prefix = if no_uv { + let prefix = if no_uv_install { "pip".to_owned() } else { py_version.to_cache_dir_top_level() @@ -1575,6 +1540,7 @@ pub async fn handle_python_reqs( .into_iter() .map(|(req, venv_p)| { let os = os.clone(); + let prefix = prefix.clone(); async move { if pull_from_tar(os, venv_p.clone(), prefix).await.is_ok() { PullFromTar::Pulled(venv_p.to_string()) @@ -1635,7 +1601,7 @@ pub async fn handle_python_reqs( let req = req.to_string(); vars.push(("REQ", &req)); vars.push(("TARGET", &venv_p)); - if !no_uv { + if !no_uv_install { vars.push(("PY_PATH", &py_path)); } let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -1808,7 +1774,7 @@ pub async fn handle_python_reqs( } else { let venv_p = venv_p.clone(); - let (cache_dir, prefix) = if no_uv { + let (cache_dir, prefix) = if no_uv_install { (PIP_CACHE_DIR.to_owned(), "pip".to_owned()) } else { py_version.to_cache_dir_tuple() From 1c1ecd6ef11346b5078f1bb23e21efba0f490d75 Mon Sep 17 00:00:00 2001 From: pyranota Date: Wed, 30 Oct 2024 19:30:37 +0000 Subject: [PATCH 23/56] Make NSJAIL work [Unsafe] config file missed /proc mount causing install phase to fail --- .../nsjail/download.py.config.proto | 19 ++++++ .../windmill-worker/src/python_executor.rs | 61 +++++++++---------- backend/windmill-worker/src/worker.rs | 6 -- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index c718f0ec302fd..71ffb9ac2f81e 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -58,6 +58,14 @@ mount { is_bind: true rw: true } +# We need it for uv +# TODO: Dont expose /proc here, it is not safe +mount { + src: "/proc" + dst: "/proc" + is_bind: true + rw: true +} mount { dst: "/tmp" @@ -79,6 +87,17 @@ mount { is_bind: true rw: true } +mount { + src: "/tmp/windmill/cache/uv" + dst: "/tmp/windmill/cache/uv" + is_bind: true + rw: true +} +mount { + src: "/tmp/windmill/cache/python" + dst: "/tmp/windmill/cache/python" + is_bind: true +} mount { src: "/dev/urandom" diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 65ee631e17b61..cd4dba1c6b03f 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -17,7 +17,10 @@ use uuid::Uuid; #[cfg(all(feature = "enterprise", feature = "parquet"))] use windmill_common::ee::{get_license_plan, LicensePlan}; use windmill_common::{ - error::{self, Error}, + error::{ + self, + Error::{self}, + }, jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, utils::calculate_hash, worker::{write_file, PythonAnnotations, WORKER_CONFIG}, @@ -82,7 +85,7 @@ use crate::{ handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, INSTANCE_PYTHON_VERSION, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, - PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, + PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR, }; #[derive(Eq, PartialEq, Clone, Copy)] @@ -97,13 +100,14 @@ impl PyVersion { /// e.g.: `/tmp/windmill/cache/python_3xy` pub fn to_cache_dir(&self) -> String { use windmill_common::worker::ROOT_CACHE_DIR; - format!("{ROOT_CACHE_DIR}python_{}", &self.to_cache_dir_top_level()) + format!("{ROOT_CACHE_DIR}{}", &self.to_cache_dir_top_level()) } /// e.g.: `python_3xy` pub fn to_cache_dir_top_level(&self) -> String { format!("python_{}", self.to_string_no_dots()) } /// e.g.: `(to_cache_dir(), to_cache_dir_top_level())` + #[cfg(all(feature = "enterprise", feature = "parquet"))] pub fn to_cache_dir_tuple(&self) -> (String, String) { use windmill_common::worker::ROOT_CACHE_DIR; let top_level = self.to_cache_dir_top_level(); @@ -163,6 +167,21 @@ impl PyVersion { occupancy_metrics: &mut Option<&mut OccupancyMetrics>, version: &str, ) -> error::Result<()> { + // Create dirs for newly installed python + // If we dont do this, NSJAIL will not be able to mount cache + // For the default version directory created during startup (main.rs) + DirBuilder::new() + .recursive(true) + .create( + PyVersion::from_string_with_dots(version) + .ok_or(error::Error::BadRequest( + "Invalid python version".to_owned(), + ))? + .to_cache_dir(), + ) + .await + .expect("could not create initial worker dir"); + let logs = String::new(); // let v_with_dot = self.to_string_with_dots(); let mut child_cmd = Command::new(UV_PATH.as_str()); @@ -403,20 +422,6 @@ pub async fn uv_pip_compile( // py-000..000-no_uv } if !no_cache { - /* - There are several scenarious of flow - 1. Cache does not exist for script - 2. Cache exists and in cached lockfile there is no python version - 3. Cache exists and in cached lockfile there is python version - Flows: - 1: We calculate lockfile and add python version to it - 2: Only possible if script exists since versions of windmill that dont support multipython - 1. If INSTANCE_PYTHON_VERSION == 3.11 assign this version to lockfile and return - 3. If INSTANCE_PYTHON_VERSION != 3.11 recalculate lockfile - 3: - 1. if cached_lockfile != annotated_version recalculate lockfile - else: return cache - */ if let Some(cached) = sqlx::query_scalar!( "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", // Python version is not included in hash, @@ -429,7 +434,7 @@ pub async fn uv_pip_compile( if let Some(line) = cached.lines().next() { if let Some(cached_version) = PyVersion::parse_lockfile(line) { // We should overwrite any version in lockfile - // And only return cache if cached_version == new_version + // And only return cache if cached_version == py_version // This will trigger recompilation for all deps, // but it is ok, since we do only on deploy and cached lockfiles for non-deployed scripts // are being cleaned up every 3 days anyway @@ -445,23 +450,12 @@ pub async fn uv_pip_compile( ); // We will assign a python version to this script } - // TODO: Small optimisation - // We end up here only if we try to redeploy script without assigned version - // So we could check if final_version == 3.11 and if so, assign python version (write to lockfile) - // and return cache - // else if new_py_version == PyVersion::Py311 { - // tracing::info!( - // "There is no assigned python version to script in job: {job_id:?}\n" - // ); - // logs.push_str(&format!("\nFound cached resolution: {req_hash}")); - // return Ok(cached); - // // We will assign a python version to this script - // } } else { tracing::error!("No requirement specified in uv_pip_compile"); } } } + let file = "requirements.in"; write_file(job_dir, file, &requirements)?; @@ -1454,6 +1448,7 @@ pub async fn handle_python_reqs( } } + let py_cache_dir = py_version.to_cache_dir(); let _ = write_file( job_dir, "download.config.proto", @@ -1468,7 +1463,7 @@ pub async fn handle_python_reqs( if no_uv_install { PIP_CACHE_DIR } else { - PY311_CACHE_DIR + &py_cache_dir }, ) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), @@ -1482,6 +1477,7 @@ pub async fn handle_python_reqs( if req.starts_with('#') { continue; } + let py_prefix = if no_uv_install { PIP_CACHE_DIR } else { @@ -1603,6 +1599,9 @@ pub async fn handle_python_reqs( vars.push(("TARGET", &venv_p)); if !no_uv_install { vars.push(("PY_PATH", &py_path)); + vars.push(("UV_PYTHON_INSTALL_DIR", "/tmp/windmill/cache/python")); + vars.push(("UV_PYTHON_PREFERENCE", "only-managed")); + vars.push(("UV_CACHE_DIR", UV_CACHE_DIR)); } let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 089e349fa2839..f324e1c151905 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -244,12 +244,6 @@ pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); -// pub fn PYX_CACHE_DIR(x: &str) -> &str { -// match x { - -// } -// } - pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); From 950e342a7f0dc1a753cf59cd51400d96631ef479 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Wed, 30 Oct 2024 20:00:40 +0000 Subject: [PATCH 24/56] Trigger CI --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 4afb307c8fadf..8354b33a0754e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# trigger CI | Dont forget to remove ARG DEBIAN_IMAGE=debian:bookworm-slim ARG RUST_IMAGE=rust:1.80-slim-bookworm ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm From 034a555f6709ce42adaea022d72123b5b3364825 Mon Sep 17 00:00:00 2001 From: pyranota Date: Wed, 30 Oct 2024 20:17:52 +0000 Subject: [PATCH 25/56] Clean up --- backend/windmill-common/src/worker.rs | 3 --- backend/windmill-worker/nsjail/download.py.config.proto | 6 ------ backend/windmill-worker/nsjail/download_deps.py.sh | 2 +- backend/windmill-worker/src/worker.rs | 6 ------ backend/windmill-worker/src/worker_lockfiles.rs | 9 ++------- 5 files changed, 3 insertions(+), 23 deletions(-) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 562adb9ff5bcc..2cceb7084fad9 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -87,9 +87,6 @@ lazy_static::lazy_static! { .ok() .and_then(|x| x.parse::().ok()) .unwrap_or(false); - - // pub static ref INSTANCE_PYTHON_VERSION: String = - // std::env::var("PYTHON_VERSION").unwrap_or_else(|_| "3.11".to_string()); } pub async fn make_suspended_pull_query(wc: &WorkerConfig) { diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index 71ffb9ac2f81e..4f3cb4448044f 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -87,12 +87,6 @@ mount { is_bind: true rw: true } -mount { - src: "/tmp/windmill/cache/uv" - dst: "/tmp/windmill/cache/uv" - is_bind: true - rw: true -} mount { src: "/tmp/windmill/cache/python" dst: "/tmp/windmill/cache/python" diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index afed8bf08d6c4..64a68c98d95d6 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -27,7 +27,7 @@ CMD="/usr/local/bin/uv pip install --no-color --no-deps --link-mode=copy --p $PY_PATH +-p \"$PY_PATH\" $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG --index-strategy unsafe-best-match --system diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index f324e1c151905..8bed1240b8c62 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -239,11 +239,6 @@ pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); // Used as fallback now pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); -pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); -pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); -pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); -pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); - pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); @@ -319,7 +314,6 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); - // pub static ref DISABLE_NSJAIL: bool = false; pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL") .ok() .and_then(|x| x.parse::().ok()) diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 91c0d7a692fc7..5bb4378fc444f 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1286,10 +1286,10 @@ async fn python_dep( annotations: PythonAnnotations, ) -> std::result::Result { create_dependencies_dir(job_dir).await; - // let mut annotated_py_version = PyVersion::from_py_annotations(annotations); let instance_version = (INSTANCE_PYTHON_VERSION.read().await.clone()).unwrap_or("3.11".to_owned()); + let final_version = PyVersion::from_py_annotations(annotations).unwrap_or( PyVersion::from_string_with_dots(&instance_version).unwrap_or_else(|| { tracing::error!( @@ -1315,10 +1315,6 @@ async fn python_dep( annotations.no_cache, ) .await; - // let final_version = annotated_py_version.unwrap_or_else(|| { - // tracing::error!("Version is supposed to be Some"); - // PyVersion::Py311 - // }); // install the dependencies to pre-fill the cache if let Ok(req) = req.as_ref() { let r = handle_python_reqs( @@ -1334,7 +1330,7 @@ async fn python_dep( occupancy_metrics, // In this case we calculate lockfile each time and it is guranteed // that version in lockfile will be equal to final_version - // So we can skip parsing of lockfile to get python version + // So we can skip parsing the lockfile returned from uv_pip_compile to get python version // and instead just use final_version final_version, annotations.no_uv || annotations.no_uv_install, @@ -1371,7 +1367,6 @@ async fn capture_dependency_job( ) -> error::Result { match job_language { ScriptLang::Python3 => { - // panic!("{}", job_raw_code); let reqs = if raw_deps { job_raw_code.to_string() } else { From 5167a98ca5d8db2000e2af5ecafaada9be18eb14 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Thu, 31 Oct 2024 09:08:30 +0000 Subject: [PATCH 26/56] Make Actions build it --- .github/workflows/docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index be0bfc771c5d4..484fe2952f051 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -10,7 +10,7 @@ env: name: Build windmill:main on: push: - branches: [main] + branches: [main, multipython] tags: ["*"] pull_request: types: [opened, synchronize, reopened] @@ -438,7 +438,7 @@ jobs: build_ee_nsjail: needs: [build_ee] runs-on: ubicloud - if: (github.event_name != 'issue_comment') || (github.event_name != 'pull_request') || (contains(github.event.comment.body, '/buildimage_nsjail') || contains(github.event.comment.body, '/buildimage_all')) + if: (github.event_name != 'issue_comment') || (contains(github.event.comment.body, '/buildimage_nsjail') || contains(github.event.comment.body, '/buildimage_all')) steps: - uses: actions/checkout@v4 with: From d25927272fd92e40d4f58fa0829a98ff1931e961 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Thu, 31 Oct 2024 10:08:32 +0000 Subject: [PATCH 27/56] Trigger CI #2 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fd809631a9604..26d765c7753ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# trigger CI | Dont forget to remove +# trigger CI #2 | Dont forget to remove ARG DEBIAN_IMAGE=debian:bookworm-slim ARG RUST_IMAGE=rust:1.80-slim-bookworm ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm From 544440f6e99a74bb96d0917b97a21240b71a3cce Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 31 Oct 2024 12:22:56 +0000 Subject: [PATCH 28/56] Update Dockerfile and clean up --- Dockerfile | 6 ++++ .../windmill-worker/src/python_executor.rs | 28 ++++++------------- .../src/lib/components/instanceSettings.ts | 2 +- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8354b33a0754e..a96355706e979 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,6 +95,10 @@ ARG WITH_POWERSHELL=true ARG WITH_KUBECTL=true ARG WITH_HELM=true ARG WITH_GIT=true +ARG DEFAULT_PYTHON_V=3.11 + +ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/python +ENV UV_PYTHON_PREFERENCE=only-managed RUN pip install --upgrade pip==24.2 @@ -162,6 +166,8 @@ ENV GO_PATH=/usr/local/go/bin/go # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv +RUN uv python install $DEFAULT_PYTHON_V + RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index cd4dba1c6b03f..f7c84a661c9ef 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1362,6 +1362,7 @@ pub async fn handle_python_reqs( tracing::warn!("Fallback to pip"); } + // TODO: In what cases it may fail? let py_path = py_version .get_python( job_dir, @@ -1372,16 +1373,9 @@ pub async fn handle_python_reqs( w_id, occupancy_metrics, ) - .await; - if let Err(ref err) = py_path { - tracing::error!("{}", err); - } - - // TODO: Refactor - let py_path = py_path?.replace('\n', ""); + .await?; if !*DISABLE_NSJAIL { - append_logs(&job_id, w_id, "\nPrepare NSJAIL\n", db).await; pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() .await @@ -1702,25 +1696,21 @@ pub async fn handle_python_reqs( tracing::debug!("pip install command: {:?}", command_args); - // panic!( - // "{:?}", - // [ - // "-x", - // &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), - // "--command", - // &command_args.join(" "), - // ] - // .join(" ") - // ); #[cfg(unix)] { + let pref = if no_uv_install { + py_version.to_string_no_dots() + } else { + "pip".to_owned() + }; + let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); flock_cmd .env_clear() .envs(envs) .args([ "-x", - &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), + &format!("{}/py{}-{}.lock", LOCK_CACHE_DIR, &pref, fssafe_req), "--command", &command_args.join(" "), ]) diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 4d69ac90b76ad..fdfdfa5d3f1f2 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -173,7 +173,7 @@ export const settings: Record = { description: 'Default python version for newly deployed scripts', key: 'instance_python_version', fieldType: 'text', - placeholder: '3.11', + placeholder: '3.10, 3.11 (Default), 3.12 or 3.13', storage: 'setting', }, { From 91f3a16d55422ba5b326f9102b093cc564cee1da Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 31 Oct 2024 12:44:32 +0000 Subject: [PATCH 29/56] Change fallbacks now there is only no_uv and NOUV --- backend/windmill-common/src/worker.rs | 3 -- .../windmill-worker/src/python_executor.rs | 45 ++++++++++--------- .../windmill-worker/src/worker_lockfiles.rs | 4 +- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 2cceb7084fad9..57f2752cc5bf8 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -308,9 +308,6 @@ fn parse_file(path: &str) -> Option { pub struct PythonAnnotations { pub no_cache: bool, pub no_uv: bool, - pub no_uv_install: bool, - pub no_uv_compile: bool, - pub no_uv_run: bool, pub py310: bool, pub py311: bool, diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index f7c84a661c9ef..0f0fd612f275f 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1,7 +1,7 @@ use std::{ collections::HashMap, process::Stdio, - sync::{Arc, RwLock}, + // sync::{Arc, RwLock}, }; use itertools::Itertools; @@ -49,17 +49,9 @@ lazy_static::lazy_static! { static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); - static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") + static ref NOUV: bool = std::env::var("NOUV") .ok().map(|flag| flag == "true").unwrap_or(false); - /// Use pip install - static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") - .ok().map(|flag| flag == "true").unwrap_or(false); - - static ref NO_UV_RUN: bool = std::env::var("NO_UV_RUN") - .ok().map(|flag| flag == "true").unwrap_or(false); - - static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); static ref EPHEMERAL_TOKEN_CMD: Option = std::env::var("EPHEMERAL_TOKEN_CMD").ok(); @@ -187,7 +179,12 @@ impl PyVersion { let mut child_cmd = Command::new(UV_PATH.as_str()); child_cmd .current_dir(job_dir) - .args(["python", "install", version]) + .args([ + "python", + "install", + version, + "--python-preference=only-managed", + ]) // TODO: Do we need these? .envs([ ("UV_PYTHON_INSTALL_DIR", "/tmp/windmill/cache/python"), @@ -272,9 +269,9 @@ impl PyVersion { w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result { - lazy_static::lazy_static! { - static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); - } + // lazy_static::lazy_static! { + // static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); + // } Self::get_python_inner( job_dir, @@ -294,7 +291,12 @@ impl PyVersion { let mut child_cmd = Command::new(UV_PATH.as_str()); let output = child_cmd .current_dir(job_dir) - .args(["python", "find", version]) + .args([ + "python", + "find", + version, + "--python-preference=only-managed", + ]) .envs([ ("UV_PYTHON_INSTALL_DIR", "/tmp/windmill/cache/python"), ("UV_PYTHON_PREFERENCE", "only-managed"), @@ -410,7 +412,7 @@ pub async fn uv_pip_compile( let mut req_hash = format!("py-{}", calculate_hash(&requirements)); - if no_uv || *USE_PIP_COMPILE { + if no_uv || *NOUV { logs.push_str(&format!("\nFallback to pip-compile (Deprecated!)")); // Set no_uv if not setted no_uv = true; @@ -668,7 +670,7 @@ pub async fn handle_python_job( .await?; let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); - let no_uv = *NO_UV_RUN | annotations.no_uv | annotations.no_uv_run; + let no_uv = *NOUV | annotations.no_uv; append_logs( &job.id, @@ -859,8 +861,7 @@ mount {{ &job.workspace_id, &mut Some(occupancy_metrics), ) - .await? - .replace('\n', ""); + .await?; let child = if !*DISABLE_NSJAIL { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -1267,7 +1268,7 @@ async fn handle_python_deps( w_id, occupancy_metrics, annotated_or_default_version, - annotations.no_uv || annotations.no_uv_compile, + annotations.no_uv, annotations.no_cache, ) .await @@ -1321,7 +1322,7 @@ async fn handle_python_deps( worker_dir, occupancy_metrics, final_version, - annotations.no_uv || annotations.no_uv_install, + annotations.no_uv, ) .await?; additional_python_paths.append(&mut venv_path); @@ -1355,7 +1356,7 @@ pub async fn handle_python_reqs( let pip_extra_index_url; let pip_index_url; - no_uv_install |= *USE_PIP_INSTALL; + no_uv_install |= *NOUV; if no_uv_install { append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index c6cd5ce0ce74d..8dc7f3c83ffc9 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1311,7 +1311,7 @@ async fn python_dep( w_id, occupancy_metrics, final_version, - annotations.no_uv || annotations.no_uv_compile, + annotations.no_uv, annotations.no_cache, ) .await; @@ -1333,7 +1333,7 @@ async fn python_dep( // So we can skip parsing the lockfile returned from uv_pip_compile to get python version // and instead just use final_version final_version, - annotations.no_uv || annotations.no_uv_install, + annotations.no_uv, ) .await; From d137cc1dafdd50157d015718da613f9127a2f8d3 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 31 Oct 2024 13:00:14 +0000 Subject: [PATCH 30/56] Expose INSTANCE_PYTHON_VERSION through env variable --- backend/windmill-common/src/global_settings.rs | 3 ++- backend/windmill-worker/src/python_executor.rs | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 5e174f6a4057e..f28f144ca8599 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -33,7 +33,7 @@ pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; -pub const ENV_SETTINGS: [&str; 51] = [ +pub const ENV_SETTINGS: [&str; 52] = [ "DISABLE_NSJAIL", "MODE", "NUM_WORKERS", @@ -56,6 +56,7 @@ pub const ENV_SETTINGS: [&str; 51] = [ "GOPRIVATE", "GOPROXY", "NETRC", + "INSTANCE_PYTHON_VERSION", "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_TRUSTED_HOST", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 0f0fd612f275f..c081672ceb9d9 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -364,8 +364,6 @@ pub async fn uv_pip_compile( worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - // If not set, will default to INSTANCE_PYTHON_VERSION - // Will always be Some after execution py_version: PyVersion, // Fallback to pip-compile. Will be removed in future mut no_uv: bool, From 6023c8bd80793025c5200d21c0630dcdeae5d44e Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 1 Nov 2024 13:24:42 +0000 Subject: [PATCH 31/56] Change namings --- .../windmill-worker/src/python_executor.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index c081672ceb9d9..de0a9b3735e25 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -96,7 +96,7 @@ impl PyVersion { } /// e.g.: `python_3xy` pub fn to_cache_dir_top_level(&self) -> String { - format!("python_{}", self.to_string_no_dots()) + format!("python_{}", self.to_string_no_dot()) } /// e.g.: `(to_cache_dir(), to_cache_dir_top_level())` #[cfg(all(feature = "enterprise", feature = "parquet"))] @@ -106,11 +106,11 @@ impl PyVersion { (format!("{ROOT_CACHE_DIR}python_{}", &top_level), top_level) } /// e.g.: `3xy` - pub fn to_string_no_dots(&self) -> String { - self.to_string_with_dots().replace('.', "") + pub fn to_string_no_dot(&self) -> String { + self.to_string_with_dot().replace('.', "") } /// e.g.: `3.xy` - pub fn to_string_with_dots(&self) -> &str { + pub fn to_string_with_dot(&self) -> &str { use PyVersion::*; match self { Py310 => "3.10", @@ -175,7 +175,7 @@ impl PyVersion { .expect("could not create initial worker dir"); let logs = String::new(); - // let v_with_dot = self.to_string_with_dots(); + // let v_with_dot = self.to_string_with_dot(); let mut child_cmd = Command::new(UV_PATH.as_str()); child_cmd .current_dir(job_dir) @@ -281,13 +281,13 @@ impl PyVersion { worker_name, w_id, occupancy_metrics, - self.to_string_with_dots(), + self.to_string_with_dot(), ) .await } async fn find_python(job_dir: &str, version: &str) -> error::Result { // let mut logs = String::new(); - // let v_with_dot = self.to_string_with_dots(); + // let v_with_dot = self.to_string_with_dot(); let mut child_cmd = Command::new(UV_PATH.as_str()); let output = child_cmd .current_dir(job_dir) @@ -545,7 +545,7 @@ pub async fn uv_pip_compile( "--cache-dir", UV_CACHE_DIR, "-p", - py_version.to_string_with_dots(), + py_version.to_string_with_dot(), ]; if no_cache { args.extend(["--no-cache"]); @@ -613,7 +613,7 @@ pub async fn uv_pip_compile( file.read_to_string(&mut req_content).await?; let lockfile = format!( "# py-{}\n{}", - py_version.to_string_with_dots(), + py_version.to_string_with_dot(), req_content .lines() .filter(|x| !x.trim_start().starts_with('#')) @@ -1698,7 +1698,7 @@ pub async fn handle_python_reqs( #[cfg(unix)] { let pref = if no_uv_install { - py_version.to_string_no_dots() + py_version.to_string_no_dot() } else { "pip".to_owned() }; From ae99619c434415bf308769811ce20ad279342284 Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 1 Nov 2024 13:25:39 +0000 Subject: [PATCH 32/56] Include py-version to requirements.in Also add comments and make code much cleaner --- .../windmill-worker/src/python_executor.rs | 141 +++++++++--------- .../windmill-worker/src/worker_lockfiles.rs | 4 + 2 files changed, 71 insertions(+), 74 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index de0a9b3735e25..fd1039c11944e 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -353,6 +353,9 @@ pub fn handle_ephemeral_token(x: String) -> String { x } +// This function only invoked during deployment of script or test run. +// And never for already deployed scripts, these have their lockfiles in PostgreSQL +// thus this function call is skipped. /// Returns lockfile and python version pub async fn uv_pip_compile( job_id: &Uuid, @@ -405,6 +408,11 @@ pub async fn uv_pip_compile( requirements.to_string() }; + // Include python version to requirements.in + // We need it because same hash based on requirements.in can get calculated even for different python versions + // To prevent from overwriting same requirements.in but with different python versions, we include version to hash + let requirements = format!("# py-{}\n{}", py_version.to_string_with_dot(), requirements); + #[cfg(feature = "enterprise")] let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?; @@ -431,28 +439,11 @@ pub async fn uv_pip_compile( .fetch_optional(db) .await? { - if let Some(line) = cached.lines().next() { - if let Some(cached_version) = PyVersion::parse_lockfile(line) { - // We should overwrite any version in lockfile - // And only return cache if cached_version == py_version - // This will trigger recompilation for all deps, - // but it is ok, since we do only on deploy and cached lockfiles for non-deployed scripts - // are being cleaned up every 3 days anyway - if py_version == cached_version { - // All good we found cache - logs.push_str(&format!("\nFound cached resolution: {req_hash}")); - return Ok(cached); - } - // Annotated version should be used, thus lockfile regenerated - } else { - tracing::info!( - "There is no assigned python version to script in job: {job_id:?}\n" - ); - // We will assign a python version to this script - } - } else { - tracing::error!("No requirement specified in uv_pip_compile"); - } + logs.push_str(&format!( + "\nFound cached resolution: {req_hash}, py: {}", + py_version.to_string_with_dot() + )); + return Ok(cached); } } @@ -670,17 +661,26 @@ pub async fn handle_python_job( let no_uv = *NOUV | annotations.no_uv; - append_logs( - &job.id, - &job.workspace_id, - format!( - "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", - py_version.to_string_with_dots() - ), - db, - ) - .await; - + if no_uv { + append_logs( + &job.id, + &job.workspace_id, + format!("\n\n--- PYTHON 3.11 (Fallback) CODE EXECUTION ---\n",), + db, + ) + .await; + } else { + append_logs( + &job.id, + &job.workspace_id, + format!( + "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", + py_version.to_string_with_dot() + ), + db, + ) + .await; + } let ( import_loader, import_base64, @@ -1214,20 +1214,13 @@ async fn handle_python_deps( .unwrap_or_else(|| vec![]) .clone(); - let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); - // let mut version = PyVersion::from_py_annotations(annotations) - // .or(PyVersion::from_string_with_dots(&*INSTANCE_PYTHON_VERSION)); - // let mut version = PyVersion::from_py_annotations(annotations) - // .or(PyVersion::from_string_with_dots(&*INSTANCE_PYTHON_VERSION)); - // Precendence: - // 1. Annotated version - // 2. Instance version - // 3. Hardcoded 3.11 + let is_deployed = requirements_o.is_some(); + let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); let instance_version = (INSTANCE_PYTHON_VERSION.read().await.clone()).unwrap_or("3.11".to_owned()); - let annotated_or_default_version = PyVersion::from_py_annotations(annotations).unwrap_or( + let annotated_or_instance_version = PyVersion::from_py_annotations(annotations).unwrap_or( PyVersion::from_string_with_dots(&instance_version).unwrap_or_else(|| { tracing::error!( "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to 3.11", @@ -1251,9 +1244,11 @@ async fn handle_python_deps( .await? .join("\n"); if requirements.is_empty() { - // TODO: "# py-3.11".to_string() - // TODO: Still check lockfile - "".to_string() + // Skip compilation step and assign py version to this execution + format!( + "# py-{}\n", + annotated_or_instance_version.to_string_with_dot() + ) } else { uv_pip_compile( job_id, @@ -1265,7 +1260,7 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - annotated_or_default_version, + annotated_or_instance_version, annotations.no_uv, annotations.no_cache, ) @@ -1277,37 +1272,29 @@ async fn handle_python_deps( } }; - // let final_version = version.unwrap_or_else(|| { - // tracing::error!("Version is supposed to be Some"); - // PyVersion::Py311 - // }); - - // Read more in next comment section - let mut final_version = PyVersion::Py311; - - if requirements.len() > 0 { + // If len > 0 it means there is at least one dependency or assigned python version + let final_version = if requirements.len() > 0 { let req: Vec<&str> = requirements .split("\n") .filter(|x| !x.starts_with("--")) .collect(); - // uv_pip_compile stage will be skipped for deployed scripts. - // Leaving us with 2 scenarious: - // 1. We have version in lockfile - // 2. We dont - // - // We want to use 3.11 version for scripts without assigned python version - // Because this means that this script was deployed before multiple python version support was introduced - // And the default version of python before this point was 3.11 - // - // But for 1. we just parse line to get version - - // Parse lockfile's line and if there is no version, fallback to annotation_default - if let Some(v) = PyVersion::parse_lockfile(&req[0]) { - final_version = v; - } - // final_version = PyVersion::parse_lockfile(&req[0]).unwrap_or(final_version); - + let final_version = if is_deployed { + // If script is deployed we can try to parse first line to get assigned version + if let Some(v) = PyVersion::parse_lockfile(&req.get(0).unwrap_or(&"")) { + // TODO: Proper error handling ^^^^^^^^^^^^^ + // Assigned + v + } else { + // If there is no assigned version in lockfile we automatically fallback to 3.11 + // In this case we have dependencies, but no associated python version + PyVersion::Py311 + } + } else { + // This is not deployed script, meaning we test run it (Preview) + // In this case we can say that desired version is `annotated_or_default` + annotated_or_instance_version + }; let mut venv_path = handle_python_reqs( req, job_id, @@ -1324,7 +1311,13 @@ async fn handle_python_deps( ) .await?; additional_python_paths.append(&mut venv_path); - } + + final_version + } else { + // If there is no assigned version in lockfile we automatically fallback to 3.11 + // In this case we no dependencies and no associated python version + PyVersion::Py311 + }; Ok((final_version, additional_python_paths)) } diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 8dc7f3c83ffc9..b246f00ed16ac 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1290,6 +1290,10 @@ async fn python_dep( let instance_version = (INSTANCE_PYTHON_VERSION.read().await.clone()).unwrap_or("3.11".to_owned()); + // Unlike `handle_python_deps` which we use for running scripts (deployed and drafts) + // This one used specifically for deploying scripts + // So we can get final_version straight away and include in lockfile + // It will be written to db as a "lock" field for script let final_version = PyVersion::from_py_annotations(annotations).unwrap_or( PyVersion::from_string_with_dots(&instance_version).unwrap_or_else(|| { tracing::error!( From 8afb4341b27205215baeb0993c9d6ea93807e64c Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 1 Nov 2024 13:47:59 +0000 Subject: [PATCH 33/56] Use const for python installation dir It was hardcoded before --- Dockerfile | 8 ++++++-- backend/windmill-worker/nsjail/download.py.config.proto | 4 ++-- backend/windmill-worker/src/python_executor.rs | 7 ++++--- backend/windmill-worker/src/worker.rs | 1 + 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3ffd17f628d3d..71e3017ddd50d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,9 +95,12 @@ ARG WITH_POWERSHELL=true ARG WITH_KUBECTL=true ARG WITH_HELM=true ARG WITH_GIT=true -ARG DEFAULT_PYTHON_V=3.11 -ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/python +# NOTE: It differs from instance version. +# `Instance` version is controllable by instance superadmins +# Where `Default` is set by default for fresh instances +ARG DEFAULT_PYTHON_V=3.11 +ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_install ENV UV_PYTHON_PREFERENCE=only-managed RUN pip install --upgrade pip==24.2 @@ -166,6 +169,7 @@ ENV GO_PATH=/usr/local/go/bin/go # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv +# Preinstall default python version RUN uv python install $DEFAULT_PYTHON_V RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index 4f3cb4448044f..78aef4fca6c33 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -88,8 +88,8 @@ mount { rw: true } mount { - src: "/tmp/windmill/cache/python" - dst: "/tmp/windmill/cache/python" + src: "{PY_INSTALL_DIR}" + dst: "{PY_INSTALL_DIR}" is_bind: true } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index fd1039c11944e..7498185b53389 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -77,7 +77,7 @@ use crate::{ handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, INSTANCE_PYTHON_VERSION, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, - PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR, + PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; #[derive(Eq, PartialEq, Clone, Copy)] @@ -187,7 +187,7 @@ impl PyVersion { ]) // TODO: Do we need these? .envs([ - ("UV_PYTHON_INSTALL_DIR", "/tmp/windmill/cache/python"), + ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), ("UV_PYTHON_PREFERENCE", "only-managed"), ]) .stdout(Stdio::piped()) @@ -298,7 +298,7 @@ impl PyVersion { "--python-preference=only-managed", ]) .envs([ - ("UV_PYTHON_INSTALL_DIR", "/tmp/windmill/cache/python"), + ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), ("UV_PYTHON_PREFERENCE", "only-managed"), ]) // .stdout(Stdio::piped()) @@ -1444,6 +1444,7 @@ pub async fn handle_python_reqs( NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT }) .replace("{WORKER_DIR}", &worker_dir) + .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) .replace( "{CACHE_DIR}", if no_uv_install { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8bed1240b8c62..38173a5a01509 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -240,6 +240,7 @@ pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); +pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_install"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); From 7cbd4bfba323ad253af497187d28e5d6c00c8c8a Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 1 Nov 2024 14:36:28 +0000 Subject: [PATCH 34/56] Pin preinstalled version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 71e3017ddd50d..1a8c1b2fb846c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,7 @@ ARG WITH_GIT=true # NOTE: It differs from instance version. # `Instance` version is controllable by instance superadmins # Where `Default` is set by default for fresh instances -ARG DEFAULT_PYTHON_V=3.11 +ARG DEFAULT_PYTHON_V=3.11.10 ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_install ENV UV_PYTHON_PREFERENCE=only-managed From 4d7faf573ac820c9cb88c60fcacc47c47bba88e0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 3 Nov 2024 02:57:48 +0100 Subject: [PATCH 35/56] Update python_executor.rs --- backend/windmill-worker/src/python_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 7948e3483627a..ee886449bf96a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -76,7 +76,7 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, PROXY_ENVS, - INSTANCE_PYTHON_VERSION, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, + INSTANCE_PYTHON_VERSION, LOCK_CACHE_DIR, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; From 82b5bbe89200ba50d38574f8ed63275c20e7ae93 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 12 Dec 2024 00:28:21 +0100 Subject: [PATCH 36/56] Up to date branch --- backend/src/main.rs | 9 +- backend/src/monitor.rs | 8 +- .../windmill-common/src/global_settings.rs | 2 +- backend/windmill-common/src/worker.rs | 1 + .../nsjail/download_deps.py.sh | 2 +- .../windmill-worker/src/ansible_executor.rs | 3 +- backend/windmill-worker/src/global_cache.rs | 21 +- .../windmill-worker/src/python_executor.rs | 280 ++++++++++-------- backend/windmill-worker/src/worker.rs | 2 +- .../windmill-worker/src/worker_lockfiles.rs | 23 +- 10 files changed, 198 insertions(+), 153 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 9f990b04d7ed5..4350bfc67834f 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -9,8 +9,8 @@ use anyhow::Context; use monitor::{ load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config, - reload_timeout_wait_result_setting, send_current_log_file_to_object_store, - send_logs_to_object_store, + reload_instance_python_version_setting, reload_timeout_wait_result_setting, + send_current_log_file_to_object_store, send_logs_to_object_store, }; use rand::Rng; use sqlx::{postgres::PgListener, Pool, Postgres}; @@ -68,8 +68,7 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR, TMP_LOGS_DIR, - UV_CACHE_DIR, + PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -1014,7 +1013,7 @@ pub async fn run_workers( TMP_LOGS_DIR, UV_CACHE_DIR, TAR_PIP_CACHE_DIR, - TAR_PY311_CACHE_DIR, + // TAR_PY311_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 3d48905118dbf..6a1312fb4621b 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -37,10 +37,10 @@ use windmill_common::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, - JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, - OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, + OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 46cbb578dfa68..b307e80e4edac 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -38,7 +38,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; -pub const ENV_SETTINGS: [&str; 54] = [ +pub const ENV_SETTINGS: [&str; 55] = [ "DISABLE_NSJAIL", "MODE", "NUM_WORKERS", diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 60027b01f7bde..5489695fb5c54 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -343,6 +343,7 @@ pub struct PythonAnnotations { pub py311: bool, pub py312: bool, pub py313: bool, + pub py_sys: bool, } #[annotations("//")] diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index 64a68c98d95d6..0e976f0e9784d 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -27,7 +27,7 @@ CMD="/usr/local/bin/uv pip install --no-color --no-deps --link-mode=copy --p \"$PY_PATH\" +$PY_PATH $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG --index-strategy unsafe-best-match --system diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index c7108d9b19c8a..dc41215c66593 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -119,8 +119,7 @@ async fn handle_ansible_python_deps( &mut Some(occupancy_metrics), crate::python_executor::PyVersion::Py311, false, - true, - true, + false, ) .await?; additional_python_paths.append(&mut venv_path); diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 975d7dc2a6b19..5ef404469dc82 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -1,4 +1,3 @@ - #[cfg(all(feature = "enterprise", feature = "parquet", unix))] use crate::PIP_CACHE_DIR; @@ -21,24 +20,24 @@ use std::sync::Arc; pub async fn build_tar_and_push( s3_client: Arc, folder: String, - // /tmp/windmill/cache/python_311 - cache_dir: String, // python_311 - prefix: String, + python_xyz: String, no_uv: bool, ) -> error::Result<()> { use object_store::path::Path; - use crate::{TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR}; + use crate::{TAR_PIP_CACHE_DIR, TAR_PYBASE_CACHE_DIR}; tracing::info!("Started building and pushing piptar {folder}"); let start = Instant::now(); + + // e.g. tiny==1.0.0 let folder_name = folder.split("/").last().unwrap(); let prefix = if no_uv { TAR_PIP_CACHE_DIR } else { - TAR_PY311_CACHE_DIR + &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz) }; let tar_path = format!("{prefix}/{folder_name}_tar.tar",); @@ -62,7 +61,7 @@ pub async fn build_tar_and_push( .put( &Path::from(format!( "/tar/{}/{folder_name}.tar", - if no_uv { "pip" } else { "python_311" } + if no_uv { "pip" } else { &python_xyz } )), std::fs::read(&tar_path)?.into(), ) @@ -87,13 +86,13 @@ pub async fn build_tar_and_push( Ok(()) } - #[cfg(all(feature = "enterprise", feature = "parquet", unix))] pub async fn pull_from_tar( client: Arc, folder: String, + // python_311 + python_xyz: String, no_uv: bool, - ) -> error::Result<()> { use windmill_common::s3_helpers::attempt_fetch_bytes; @@ -105,12 +104,10 @@ pub async fn pull_from_tar( let tar_path = format!( "tar/{}/{folder_name}.tar", - if no_uv { "pip" } else { "python_311" } + if no_uv { "pip".to_owned() } else { python_xyz } ); let bytes = attempt_fetch_bytes(client, &tar_path).await?; - // tracing::info!("B: {target} {folder}"); - extract_tar(bytes, &folder).await.map_err(|e| { tracing::error!("Failed to extract piptar {folder_name}. Error: {:?}", e); e diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index e2eacbcadd952..cba99eec90f94 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -57,11 +57,10 @@ lazy_static::lazy_static! { static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); - static ref NOUV: bool = std::env::var("NOUV") + pub static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") .ok().map(|flag| flag == "true").unwrap_or(false); - /// Use pip install - static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") + pub static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") .ok().map(|flag| flag == "true").unwrap_or(false); static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); @@ -87,9 +86,9 @@ use crate::{ read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR, - NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, - PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, + AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, + LOCK_CACHE_DIR, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, + PROXY_ENVS, PY311_CACHE_DIR, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; #[derive(Eq, PartialEq, Clone, Copy)] @@ -98,6 +97,7 @@ pub enum PyVersion { Py311, Py312, Py313, + System, } impl PyVersion { @@ -129,6 +129,7 @@ impl PyVersion { Py311 => "3.11", Py312 => "3.12", Py313 => "3.13", + System => "sys", } } pub fn from_string_with_dots(value: &str) -> Option { @@ -138,15 +139,16 @@ impl PyVersion { "3.11" => Some(Py311), "3.12" => Some(Py312), "3.13" => Some(Py313), + "sys" => Some(System), _ => None, } } /// e.g.: `# py-3.xy` -> `PyVersion::Py3XY` - pub fn parse_lockfile(line: &str) -> Option { + pub fn parse_version(line: &str) -> Option { Self::from_string_with_dots(line.replace("# py-", "").as_str()) } pub fn from_py_annotations(a: PythonAnnotations) -> Option { - let PythonAnnotations { py310, py311, py312, py313, .. } = a; + let PythonAnnotations { py_sys, py310, py311, py312, py313, .. } = a; use PyVersion::*; if py313 { Some(Py313) @@ -156,6 +158,8 @@ impl PyVersion { Some(Py311) } else if py310 { Some(Py310) + } else if py_sys { + Some(System) } else { None } @@ -198,10 +202,7 @@ impl PyVersion { "--python-preference=only-managed", ]) // TODO: Do we need these? - .envs([ - ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), - ("UV_PYTHON_PREFERENCE", "only-managed"), - ]) + .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -234,7 +235,7 @@ impl PyVersion { w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, version: &str, - ) -> error::Result { + ) -> error::Result> { let py_path = Self::find_python(job_dir, version).await; // Python is not installed @@ -280,7 +281,10 @@ impl PyVersion { worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - ) -> error::Result { + ) -> error::Result> { + if matches!(self, Self::System) { + return Ok(None); + } // lazy_static::lazy_static! { // static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); // } @@ -297,7 +301,7 @@ impl PyVersion { ) .await } - async fn find_python(job_dir: &str, version: &str) -> error::Result { + async fn find_python(job_dir: &str, version: &str) -> error::Result> { // let mut logs = String::new(); // let v_with_dot = self.to_string_with_dot(); let mut child_cmd = Command::new(UV_PATH.as_str()); @@ -323,7 +327,7 @@ impl PyVersion { // Convert the output to a String let stdout = String::from_utf8(output.stdout).expect("Failed to convert output to String"); - return Ok(stdout.replace('\n', "")); + return Ok(Some(stdout.replace('\n', ""))); } else { // If the command failed, print the error let stderr = @@ -380,17 +384,15 @@ pub async fn uv_pip_compile( w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, py_version: PyVersion, - // Fallback to pip-compile. Will be removed in future - mut no_uv: bool, // Debug-only flag no_cache: bool, + // Fallback to pip-compile. Will be removed in future + mut no_uv: bool, ) -> error::Result { let mut logs = String::new(); logs.push_str(&format!("\nresolving dependencies...")); logs.push_str(&format!("\ncontent of requirements:\n{}\n", requirements)); - // New version, the version what we wanna have - let requirements = if let Some(pip_local_dependencies) = WORKER_CONFIG.read().await.pip_local_dependencies.as_ref() { @@ -430,7 +432,7 @@ pub async fn uv_pip_compile( let mut req_hash = format!("py-{}", calculate_hash(&requirements)); - if no_uv || *NOUV { + if no_uv || *USE_PIP_COMPILE { logs.push_str(&format!("\nFallback to pip-compile (Deprecated!)")); // Set no_uv if not setted no_uv = true; @@ -444,15 +446,15 @@ pub async fn uv_pip_compile( if !no_cache { if let Some(cached) = sqlx::query_scalar!( "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", - // Python version is not included in hash, - // meaning hash will stay the same independant from python version + // Python version is included in hash, + // hash will be the different for every python version req_hash ) .fetch_optional(db) .await? { logs.push_str(&format!( - "\nFound cached resolution: {req_hash}, py: {}", + "\nFound cached resolution: {req_hash}, on python version: {}", py_version.to_string_with_dot() )); return Ok(cached); @@ -550,9 +552,17 @@ pub async fn uv_pip_compile( // Target to /tmp/windmill/cache/uv "--cache-dir", UV_CACHE_DIR, - "-p", - py_version.to_string_with_dot(), ]; + if !matches!(py_version, PyVersion::System) { + args.extend([ + "-p", + py_version.to_string_with_dot(), + "--python-preference", + "only-managed", + ]); + } else { + args.extend(["--python-preference", "only-system"]); + } if no_cache { args.extend(["--no-cache"]); } @@ -638,8 +648,6 @@ pub async fn uv_pip_compile( sqlx::query!( "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", req_hash, - // format!("# py-{}\n{lockfile}",py_version.unwrap_or(&*INSTANCE_PYTHON_VERSION)) - // format!("# py-{}\n{lockfile}",py_version) lockfile ).fetch_optional(db).await?; @@ -793,7 +801,7 @@ pub async fn handle_python_job( ) -> windmill_common::error::Result> { let script_path = crate::common::use_flow_root_path(job.script_path()); - let (py_version, additional_python_paths) = handle_python_deps( + let (py_version, mut additional_python_paths) = handle_python_deps( job_dir, requirements_o, inner_content, @@ -808,11 +816,22 @@ pub async fn handle_python_job( &mut Some(occupancy_metrics), ) .await?; - let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); + let PythonAnnotations { + no_cache, + no_uv, + no_uv_install, + no_uv_compile, + no_postinstall, + py310, + py311, + py312, + py313, + py_sys, + } = PythonAnnotations::parse(inner_content); tracing::debug!("Finished handling python dependencies"); - if !PythonAnnotations::parse(inner_content).no_postinstall { + if !no_postinstall { if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, db).await { tracing::error!("Postinstall stage has failed. Reason: {e}"); } @@ -827,7 +846,6 @@ pub async fn handle_python_job( ) .await; - if no_uv { append_logs( &job.id, @@ -1021,7 +1039,9 @@ mount {{ job.id ); - let python_path = py_version + let python_path = if no_uv { + PYTHON_PATH.clone() + } else if let Some(python_path) = py_version .get_python( job_dir, &job.id, @@ -1031,7 +1051,12 @@ mount {{ &job.workspace_id, &mut Some(occupancy_metrics), ) - .await?; + .await? + { + python_path + } else { + PYTHON_PATH.clone() + }; let child = if !*DISABLE_NSJAIL { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -1049,11 +1074,7 @@ mount {{ "--config", "run.config.proto", "--", - if no_uv { - PYTHON_PATH.as_str() - } else { - &python_path - }, + &python_path, "-u", "-m", "wrapper", @@ -1062,13 +1083,7 @@ mount {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - // let mut python_cmd = Command::new(PYTHON_PATH.as_str()); - // tracing::error!("{}", python_path); - let mut python_cmd = if no_uv { - Command::new(PYTHON_PATH.as_str()) - } else { - Command::new(python_path.as_str()) - }; + let mut python_cmd = Command::new(&python_path); let args = vec!["-u", "-m", "wrapper"]; python_cmd @@ -1090,11 +1105,7 @@ mount {{ python_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); } - if no_uv { - start_child_process(python_cmd, PYTHON_PATH.as_str()).await? - } else { - start_child_process(python_cmd, python_path.as_str()).await? - } + start_child_process(python_cmd, &python_path).await? }; handle_child( @@ -1390,27 +1401,26 @@ async fn handle_python_deps( .unwrap_or_else(|| vec![]) .clone(); - let is_deployed = requirements_o.is_some(); - let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); - let instance_version = - (INSTANCE_PYTHON_VERSION.read().await.clone()).unwrap_or("3.11".to_owned()); - - let annotated_or_instance_version = PyVersion::from_py_annotations(annotations).unwrap_or( - PyVersion::from_string_with_dots(&instance_version).unwrap_or_else(|| { - tracing::error!( + let instance_version = INSTANCE_PYTHON_VERSION + .read() + .await + .clone() + .and_then(|v| PyVersion::from_string_with_dots(&v)) + .unwrap_or_else(|| { + tracing::warn!( "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to 3.11", *INSTANCE_PYTHON_VERSION ); PyVersion::Py311 - }), - ); - - let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); + }); + let annotated_version = PyVersion::from_py_annotations(annotations); + let mut is_deployed = true; let requirements = match requirements_o { Some(r) => r, None => { + is_deployed = false; let mut already_visited = vec![]; let requirements = windmill_parser_py_imports::parse_python_imports( @@ -1422,12 +1432,9 @@ async fn handle_python_deps( ) .await? .join("\n"); + if requirements.is_empty() { - // Skip compilation step and assign py version to this execution - format!( - "# py-{}\n", - annotated_or_instance_version.to_string_with_dot() - ) + "".to_owned() } else { uv_pip_compile( job_id, @@ -1439,10 +1446,9 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - annotated_or_instance_version, - annotations.no_uv, - annotations.no_uv || annotations.no_uv_compile, + annotated_version.unwrap_or(instance_version), annotations.no_cache, + annotations.no_uv || annotations.no_uv_compile, ) .await .map_err(|e| { @@ -1451,35 +1457,53 @@ async fn handle_python_deps( } } }; - - // If len > 0 it means there is at least one dependency or assigned python version - let final_version = if requirements.len() > 0 { - let req: Vec<&str> = requirements + /* + For deployed scripts we want to find out version in following order: + 1. Annotated version + 2. Assigned version (written in lockfile) + 3. 3.11 + + For Previews: + 1. Annotated version + 2. Instance version + 3. 3.11 + */ + + let requirements_lines: Vec<&str> = if requirements.len() > 0 { + requirements .split("\n") - .filter(|x| !x.starts_with("--")) - .collect(); + .filter(|x| !x.starts_with("--") && !x.trim().is_empty()) + .collect() + } else { + vec![] + }; - let final_version = if is_deployed { + let final_version = annotated_version.unwrap_or_else(|| { + if is_deployed { // If script is deployed we can try to parse first line to get assigned version - if let Some(v) = PyVersion::parse_lockfile(&req.get(0).unwrap_or(&"")) { - // TODO: Proper error handling ^^^^^^^^^^^^^ - // Assigned + if let Some(v) = requirements_lines + .get(0) + .and_then(|line| PyVersion::parse_version(line)) + { + // We have assigned version, we should use it v } else { // If there is no assigned version in lockfile we automatically fallback to 3.11 // In this case we have dependencies, but no associated python version + // This is the case for old deployed scripts PyVersion::Py311 } } else { // This is not deployed script, meaning we test run it (Preview) - // In this case we can say that desired version is `annotated_or_default` - annotated_or_instance_version - }; + // In this case we can say that desired version is `instance_version` + instance_version + } + }); + + // If len > 0 it means there is atleast one dependency or assigned python version + if requirements.len() > 0 { let mut venv_path = handle_python_reqs( - requirements - .split("\n") - .filter(|x| !x.starts_with("--") && !x.trim().is_empty()) - .collect(), + requirements_lines, job_id, w_id, mem_peak, @@ -1490,20 +1514,14 @@ async fn handle_python_deps( worker_dir, occupancy_metrics, final_version, - annotations.no_uv, - annotations.no_uv || annotations.no_uv_install, false, - + annotations.no_uv || annotations.no_uv_install, ) .await?; additional_python_paths.append(&mut venv_path); + } - final_version - } else { - // If there is no assigned version in lockfile we automatically fallback to 3.11 - // In this case we no dependencies and no associated python version - PyVersion::Py311 - }; + // TODO: Annotated version should always be equal to final_version Ok((final_version, additional_python_paths)) } @@ -1515,23 +1533,21 @@ lazy_static::lazy_static! { /// Can be wrapped by nsjail depending on configuration #[inline] async fn spawn_uv_install( - w_id: &str, req: &str, venv_p: &str, job_dir: &str, - (pip_extra_index_url, pip_index_url): (Option, Option), + // If none, it is system python + py_path: Option, no_uv_install: bool, ) -> Result { - if !*DISABLE_NSJAIL { tracing::info!( workspace_id = %w_id, "starting nsjail" ); - let mut vars = vec![("PATH", PATH_ENV.as_str())]; if let Some(url) = pip_extra_index_url.as_ref() { vars.push(("EXTRA_INDEX_URL", url)); @@ -1545,7 +1561,14 @@ async fn spawn_uv_install( if let Some(host) = PIP_TRUSTED_HOST.as_ref() { vars.push(("TRUSTED_HOST", host)); } - + let _owner; + if let Some(py_path) = py_path.as_ref() { + _owner = format!( + "-p {} --python-preference only-managed", + py_path.as_str() // + ); + vars.push(("PY_PATH", &_owner)); + } vars.push(("REQ", &req)); vars.push(("TARGET", venv_p)); @@ -1595,8 +1618,6 @@ async fn spawn_uv_install( &req, "--no-deps", "--no-color", - // "-p", - // "3.11", // Prevent uv from discovering configuration files. "--no-config", "--link-mode=copy", @@ -1613,6 +1634,20 @@ async fn spawn_uv_install( ] }; + if let Some(py_path) = py_path.as_ref() { + command_args.extend([ + "-p", + py_path.as_str(), + "--python-preference", + "only-managed", // + ]); + } else { + command_args.extend([ + "--python-preference", + "only-system", // + ]); + } + if let Some(url) = pip_extra_index_url.as_ref() { url.split(",").for_each(|url| { command_args.extend(["--extra-index-url", url]); @@ -1702,7 +1737,7 @@ fn pad_string(value: &str, total_length: usize) -> String { } } -/// pip install, include cached or pull from S3 +/// uv pip install, include cached or pull from S3 pub async fn handle_python_reqs( requirements: Vec<&str>, job_id: &Uuid, @@ -1714,6 +1749,7 @@ pub async fn handle_python_reqs( job_dir: &str, worker_dir: &str, _occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + py_version: PyVersion, // TODO: Remove (Deprecated) mut no_uv_install: bool, is_ansible: bool, @@ -1815,11 +1851,11 @@ pub async fn handle_python_reqs( .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) .replace( "{CACHE_DIR}", - if no_uv_install { - PIP_CACHE_DIR + &(if no_uv_install { + PIP_CACHE_DIR.to_owned() } else { - PY311_CACHE_DIR - }, + py_version.to_cache_dir() + }), ) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; @@ -1833,16 +1869,14 @@ pub async fn handle_python_reqs( // If so, skip them let mut in_cache = vec![]; for req in requirements { - // Ignore python version annotation backed into lockfile if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { continue; } - // TODO: Remove let py_prefix = if no_uv_install { PIP_CACHE_DIR } else { - PY311_CACHE_DIR + &py_version.to_cache_dir() }; let venv_p = format!( @@ -1867,7 +1901,6 @@ pub async fn handle_python_reqs( .await; } - let (kill_tx, ..) = tokio::sync::broadcast::channel::<()>(1); let kill_rxs: Vec> = (0..req_with_penv.len()) .map(|_| kill_tx.subscribe()) @@ -1980,6 +2013,19 @@ pub async fn handle_python_reqs( let is_not_pro = !matches!(get_license_plan().await, LicensePlan::Pro); let total_time = std::time::Instant::now(); + let py_path = py_version + .get_python( + &job_dir, + job_id, + _mem_peak, + db, + _worker_name, + w_id, + _occupancy_metrics, + ) + .await + .unwrap(); + let has_work = req_with_penv.len() > 0; for ((req, venv_p), mut kill_rx) in req_with_penv.iter().zip(kill_rxs.into_iter()) { let permit = semaphore.clone().acquire_owned().await; // Acquire a permit @@ -2007,6 +2053,7 @@ pub async fn handle_python_reqs( let venv_p = venv_p.clone(); let counter_arc = counter_arc.clone(); let pip_indexes = pip_indexes.clone(); + let py_path = py_path.clone(); handles.push(task::spawn(async move { // permit will be dropped anyway if this thread exits at any point @@ -2028,7 +2075,7 @@ pub async fn handle_python_reqs( tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")), - pull = pull_from_tar(os, venv_p.clone(), no_uv_install) => { + pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), no_uv_install) => { if let Err(e) = pull { tracing::info!( workspace_id = %w_id, @@ -2060,7 +2107,8 @@ pub async fn handle_python_reqs( &venv_p, &job_dir, pip_indexes, - no_uv_install, + py_path, + no_uv_install ).await { Ok(r) => r, Err(e) => { @@ -2149,7 +2197,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if s3_push { if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { - tokio::spawn(build_tar_and_push(os, venv_p.clone(), no_uv_install)); + tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), no_uv_install)); } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b2dae84df994f..c4feb18d971cc 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -259,8 +259,8 @@ pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_install"); +pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); -pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); pub const DENO_CACHE_DIR_NPM: &str = concatcp!(ROOT_CACHE_DIR, "deno/npm"); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index bacfcb512a85d..50501f61c6878 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -29,7 +29,7 @@ use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::common::OccupancyMetrics; use crate::python_executor::{ - create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, + create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, USE_PIP_INSTALL, }; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; use crate::INSTANCE_PYTHON_VERSION; @@ -1584,10 +1584,16 @@ async fn python_dep( let instance_version = (INSTANCE_PYTHON_VERSION.read().await.clone()).unwrap_or("3.11".to_owned()); - // Unlike `handle_python_deps` which we use for running scripts (deployed and drafts) - // This one used specifically for deploying scripts - // So we can get final_version straight away and include in lockfile - // It will be written to db as a "lock" field for script + /* + Unlike `handle_python_deps` which we use for running scripts (deployed and drafts) + This one used specifically for deploying scripts + So we can get final_version right away and include in lockfile + And the precendence is following: + + 1. Annotation version + 2. Instance version + 3. 311 + */ let final_version = PyVersion::from_py_annotations(annotations).unwrap_or( PyVersion::from_string_with_dots(&instance_version).unwrap_or_else(|| { tracing::error!( @@ -1626,13 +1632,8 @@ async fn python_dep( job_dir, worker_dir, occupancy_metrics, - // In this case we calculate lockfile each time and it is guranteed - // that version in lockfile will be equal to final_version - // So we can skip parsing the lockfile returned from uv_pip_compile to get python version - // and instead just use final_version final_version, - annotations.no_uv, - false, + annotations.no_uv || *USE_PIP_INSTALL, false, ) .await; From ffb20396b33669a498f31d387360fdac155f7ceb Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 12 Dec 2024 16:11:23 +0100 Subject: [PATCH 37/56] Create PYCACHE dirs TODO: PY_TAR_DIRS --- backend/src/main.rs | 8 ++++++-- backend/windmill-worker/src/python_executor.rs | 2 +- backend/windmill-worker/src/worker.rs | 7 ++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 4350bfc67834f..d1fce51ca4dbc 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -68,7 +68,8 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, + PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, PYSYS_CACHE_DIR, + RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -1013,12 +1014,15 @@ pub async fn run_workers( TMP_LOGS_DIR, UV_CACHE_DIR, TAR_PIP_CACHE_DIR, - // TAR_PY311_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, BUN_CACHE_DIR, + PY310_CACHE_DIR, PY311_CACHE_DIR, + PY312_CACHE_DIR, + PY313_CACHE_DIR, + PYSYS_CACHE_DIR, PIP_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_BUNDLE_CACHE_DIR, diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index cba99eec90f94..a1724765e890b 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -88,7 +88,7 @@ use crate::{ handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, LOCK_CACHE_DIR, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, - PROXY_ENVS, PY311_CACHE_DIR, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, + PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; #[derive(Eq, PartialEq, Clone, Copy)] diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index c4feb18d971cc..9b3bdd23dc939 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -252,10 +252,11 @@ pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); // Used as fallback now pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); -// pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); +pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); -// pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); -// pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); +pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); +pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); +pub const PYSYS_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_sys"); pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_install"); From 4b1b1b4b0c9b40b74e518ce1e3e35b4167531b0d Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 12 Dec 2024 16:32:06 +0100 Subject: [PATCH 38/56] Fix after merge --- .../windmill-worker/src/ansible_executor.rs | 1 - .../windmill-worker/src/python_executor.rs | 34 ++++++++----------- .../windmill-worker/src/worker_lockfiles.rs | 12 +++---- 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index dc41215c66593..f3e1a0ce45e1a 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -119,7 +119,6 @@ async fn handle_ansible_python_deps( &mut Some(occupancy_metrics), crate::python_executor::PyVersion::Py311, false, - false, ) .await?; additional_python_paths.append(&mut venv_path); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 26f31208af5c4..da0911fffdf43 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -88,12 +88,7 @@ use crate::{ handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, LOCK_CACHE_DIR, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, -<<<<<<< HEAD PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, -======= - PROXY_ENVS, PY311_CACHE_DIR, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, - ->>>>>>> 96a5663eb24daf5649889c045737a8c7bd372565 }; #[derive(Eq, PartialEq, Clone, Copy)] @@ -1454,8 +1449,8 @@ async fn handle_python_deps( w_id, occupancy_metrics, annotated_version.unwrap_or(instance_version), - annotations.no_cache, annotations.no_uv || annotations.no_uv_compile, + annotations.no_cache, ) .await .map_err(|e| { @@ -1521,7 +1516,6 @@ async fn handle_python_deps( worker_dir, occupancy_metrics, final_version, - false, annotations.no_uv || annotations.no_uv_install, ) .await?; @@ -1759,7 +1753,6 @@ pub async fn handle_python_reqs( py_version: PyVersion, // TODO: Remove (Deprecated) mut no_uv_install: bool, - is_ansible: bool, ) -> error::Result> { let counter_arc = Arc::new(tokio::sync::Mutex::new(0)); // Append logs with line like this: @@ -1811,7 +1804,7 @@ pub async fn handle_python_reqs( } no_uv_install |= *USE_PIP_INSTALL; - if no_uv_install && !is_ansible { + if no_uv_install { append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await; tracing::warn!("Fallback to pip"); } @@ -1934,12 +1927,12 @@ pub async fn handle_python_reqs( let mut local_mem_peak = 0; for pid_o in pids.lock().await.iter() { if pid_o.is_some(){ - let mem = get_mem_peak(*pid_o, !*DISABLE_NSJAIL).await; + let mem = crate::handle_child::get_mem_peak(*pid_o, !*DISABLE_NSJAIL).await; if mem < 0 { tracing::warn!( workspace_id = %w_id_2, - "Cannot get memory peak for pid: {:?}, job_id: {:?}, exit code: {mem}", - pid_o, + "Cannot get memory peak for pid: {:?}, job_id: {:?}, exit code: {mem}", + pid_o, job_id_2 ); } else { @@ -1956,12 +1949,12 @@ pub async fn handle_python_reqs( } else { tracing::debug!( workspace_id = %w_id_2, - "Local mem_peak {:?}mb is smaller then global one {:?}mb, ignoring. job_id: {:?}", + "Local mem_peak {:?}mb is smaller then global one {:?}mb, ignoring. job_id: {:?}", local_mem_peak / 1000, *mem_peak_lock / 1000, job_id_2 ); - + } // Get the copy of value and drop lock itself, to release it as fast as possible *mem_peak_lock @@ -1996,28 +1989,27 @@ pub async fn handle_python_reqs( if canceled { tracing::info!( - // If there is listener on other side, + // If there is listener on other side, workspace_id = %w_id_2, "cancelling installations", ); if let Err(ref e) = kill_tx.send(()){ tracing::error!( - // If there is listener on other side, + // If there is listener on other side, workspace_id = %w_id_2, "failed to send done: Probably receiving end closed too early or have not opened yet\n{}", // If there is no listener, it will be dropped safely e ); } - } + } } // Once done_tx is dropped, this will be fired _ = done_rx.recv() => break } } }); - } // tl = total_length @@ -2070,7 +2062,7 @@ pub async fn handle_python_reqs( .get_python( &job_dir, job_id, - _mem_peak, + mem_peak, db, _worker_name, w_id, @@ -2080,7 +2072,9 @@ pub async fn handle_python_reqs( .unwrap(); let has_work = req_with_penv.len() > 0; - for ((i, (req, venv_p)), mut kill_rx) in req_with_penv.iter().enumerate().zip(kill_rxs.into_iter()) { + for ((i, (req, venv_p)), mut kill_rx) in + req_with_penv.iter().enumerate().zip(kill_rxs.into_iter()) + { let permit = semaphore.clone().acquire_owned().await; // Acquire a permit if let Err(_) = permit { diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index a805ce5f6a22d..9ee1941b09594 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -29,7 +29,8 @@ use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::common::OccupancyMetrics; use crate::python_executor::{ - create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, USE_PIP_INSTALL, USE_PIP_COMPILE, + create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, USE_PIP_COMPILE, + USE_PIP_INSTALL, }; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; use crate::INSTANCE_PYTHON_VERSION; @@ -1617,8 +1618,8 @@ async fn python_dep( w_id, occupancy_metrics, final_version, - annotations.no_uv, annotations.no_cache, + no_uv_compile, ) .await; // install the dependencies to pre-fill the cache @@ -1635,9 +1636,7 @@ async fn python_dep( worker_dir, occupancy_metrics, final_version, - annotations.no_uv || *USE_PIP_INSTALL, no_uv_install, - false, ) .await; @@ -1687,8 +1686,8 @@ async fn capture_dependency_job( .join("\n") }; - let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = - PythonAnnotations::parse(job_raw_code); + let annotations = PythonAnnotations::parse(job_raw_code); + let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = annotations; if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL { if let Err(e) = sqlx::query!( @@ -1719,7 +1718,6 @@ async fn capture_dependency_job( annotations, no_uv_compile | no_uv, no_uv_install | no_uv, - ) .await } From 6e0b932e4da5795a815ed58da8d2ac25580286e7 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 12 Dec 2024 17:17:10 +0100 Subject: [PATCH 39/56] Make it safer --- backend/src/main.rs | 8 ++++- .../windmill-worker/src/python_executor.rs | 31 +++++++++---------- backend/windmill-worker/src/worker.rs | 10 +++++- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index d1fce51ca4dbc..5a9407d4347f9 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -69,7 +69,8 @@ use windmill_worker::{ BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, PYSYS_CACHE_DIR, - RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, + RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, + TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, TAR_PYSYS_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -1023,6 +1024,11 @@ pub async fn run_workers( PY312_CACHE_DIR, PY313_CACHE_DIR, PYSYS_CACHE_DIR, + TAR_PY310_CACHE_DIR, + TAR_PY311_CACHE_DIR, + TAR_PY312_CACHE_DIR, + TAR_PY313_CACHE_DIR, + TAR_PYSYS_CACHE_DIR, PIP_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_BUNDLE_CACHE_DIR, diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index da0911fffdf43..dfe46b7a702b8 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -289,7 +289,7 @@ impl PyVersion { // static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); // } - Self::get_python_inner( + let res = Self::get_python_inner( job_dir, job_id, mem_peak, @@ -299,7 +299,15 @@ impl PyVersion { occupancy_metrics, self.to_string_with_dot(), ) - .await + .await; + + if let Err(ref e) = res { + tracing::error!( + "worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n + Error while getting python from uv, falling back to system python: {e:?}" + ); + } + res } async fn find_python(job_dir: &str, version: &str) -> error::Result> { // let mut logs = String::new(); @@ -818,19 +826,8 @@ pub async fn handle_python_job( &mut Some(occupancy_metrics), ) .await?; - let PythonAnnotations { - no_cache, - no_uv, - no_uv_install, - no_uv_compile, - no_postinstall, - py310, - py311, - py312, - py313, - py_sys, - } = PythonAnnotations::parse(inner_content); + let PythonAnnotations { no_uv, no_postinstall, .. } = PythonAnnotations::parse(inner_content); tracing::debug!("Finished handling python dependencies"); if !no_postinstall { @@ -1053,7 +1050,8 @@ mount {{ &job.workspace_id, &mut Some(occupancy_metrics), ) - .await? + .await + .unwrap_or(Some(PYTHON_PATH.clone())) { python_path } else { @@ -2069,7 +2067,8 @@ pub async fn handle_python_reqs( _occupancy_metrics, ) .await - .unwrap(); + .ok() + .flatten(); let has_work = req_with_penv.len() > 0; for ((i, (req, venv_p)), mut kill_rx) in diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 398bcd075574e..76c9f4b0bddc0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -260,6 +260,12 @@ pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); pub const PYSYS_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_sys"); +pub const TAR_PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_310"); +pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"); +pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312"); +pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313"); +pub const TAR_PYSYS_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_sys"); + pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_install"); pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar"); @@ -2375,7 +2381,9 @@ async fn handle_code_execution_job( .await; } else if language == Some(ScriptLang::Mysql) { #[cfg(not(feature = "mysql"))] - return Err(Error::InternalErr("MySQL requires the mysql feature to be enabled".to_string())); + return Err(Error::InternalErr( + "MySQL requires the mysql feature to be enabled".to_string(), + )); #[cfg(feature = "mysql")] return do_mysql( From e399aa0b8d7d7bd09d26ecaa62ae866104a0d0d4 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 12 Dec 2024 17:48:07 +0100 Subject: [PATCH 40/56] Implement USE_SYSTEM_PYTHON --- .../windmill-worker/src/python_executor.rs | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index dfe46b7a702b8..d8043b84a57f5 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -38,8 +38,6 @@ use windmill_common::variables::get_secret_value_as_admin; use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { - - // Default python static ref PYTHON_PATH: String = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); @@ -57,6 +55,9 @@ lazy_static::lazy_static! { static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); + pub static ref USE_SYSTEM_PYTHON: bool = std::env::var("USE_SYSTEM_PYTHON") + .ok().map(|flag| flag == "true").unwrap_or(false); + pub static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") .ok().map(|flag| flag == "true").unwrap_or(false); @@ -282,7 +283,7 @@ impl PyVersion { w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result> { - if matches!(self, Self::System) { + if matches!(self, Self::System) || *USE_SYSTEM_PYTHON { return Ok(None); } // lazy_static::lazy_static! { @@ -1050,8 +1051,7 @@ mount {{ &job.workspace_id, &mut Some(occupancy_metrics), ) - .await - .unwrap_or(Some(PYTHON_PATH.clone())) + .await? { python_path } else { @@ -1633,18 +1633,20 @@ async fn spawn_uv_install( ] }; - if let Some(py_path) = py_path.as_ref() { - command_args.extend([ - "-p", - py_path.as_str(), - "--python-preference", - "only-managed", // - ]); - } else { - command_args.extend([ - "--python-preference", - "only-system", // - ]); + if !no_uv_install { + if let Some(py_path) = py_path.as_ref() { + command_args.extend([ + "-p", + py_path.as_str(), + "--python-preference", + "only-managed", // + ]); + } else { + command_args.extend([ + "--python-preference", + "only-system", // + ]); + } } if let Some(url) = pip_extra_index_url.as_ref() { @@ -2066,9 +2068,7 @@ pub async fn handle_python_reqs( w_id, _occupancy_metrics, ) - .await - .ok() - .flatten(); + .await?; let has_work = req_with_penv.len() > 0; for ((i, (req, venv_p)), mut kill_rx) in From 84bc1a8a0167fc02c8a589e16a65df4a27ef4d7b Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 13 Dec 2024 12:43:16 +0100 Subject: [PATCH 41/56] Implement latest_stable option --- Dockerfile | 11 +- backend/src/main.rs | 2 - backend/windmill-common/src/worker.rs | 2 +- .../windmill-worker/src/python_executor.rs | 112 ++++++++++-------- backend/windmill-worker/src/worker.rs | 17 ++- .../windmill-worker/src/worker_lockfiles.rs | 14 +-- .../src/lib/components/InstanceSetting.svelte | 12 +- .../src/lib/components/instanceSettings.ts | 20 ++-- 8 files changed, 106 insertions(+), 84 deletions(-) diff --git a/Dockerfile b/Dockerfile index 68446c4547f34..f5929ee8e7117 100644 --- a/Dockerfile +++ b/Dockerfile @@ -96,10 +96,11 @@ ARG WITH_KUBECTL=true ARG WITH_HELM=true ARG WITH_GIT=true -# NOTE: It differs from instance version. -# `Instance` version is controllable by instance superadmins -# Where `Default` is set by default for fresh instances -ARG DEFAULT_PYTHON_V=3.11.10 +# To change latest stable version: +# 1. Change placeholder in instanceSettings.ts +# 2. Change LATEST_STABLE_PY in dockerfile +# 3. Change #[default] annotation for PyVersion in backend +ARG LATEST_STABLE_PY=3.11.10 ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_install ENV UV_PYTHON_PREFERENCE=only-managed @@ -170,7 +171,7 @@ ENV GO_PATH=/usr/local/go/bin/go RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv # Preinstall default python version -RUN uv python install $DEFAULT_PYTHON_V +RUN uv python install $LATEST_STABLE_PY RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \ diff --git a/backend/src/main.rs b/backend/src/main.rs index 5a9407d4347f9..84afb8e3c5ee9 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1023,12 +1023,10 @@ pub async fn run_workers( PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, - PYSYS_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, - TAR_PYSYS_CACHE_DIR, PIP_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_BUNDLE_CACHE_DIR, diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 5489695fb5c54..4c4a65d011abf 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -339,11 +339,11 @@ pub struct PythonAnnotations { pub no_uv_install: bool, pub no_uv_compile: bool, pub no_postinstall: bool, + pub py_latest_stable: bool, pub py310: bool, pub py311: bool, pub py312: bool, pub py313: bool, - pub py_sys: bool, } #[annotations("//")] diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index d8043b84a57f5..9eaf30cb6ab43 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -92,16 +92,35 @@ use crate::{ PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; -#[derive(Eq, PartialEq, Clone, Copy)] +// To change latest stable version: +// 1. Change placeholder in instanceSettings.ts +// 2. Change LATEST_STABLE_PY in dockerfile +// 3. Change #[default] annotation for PyVersion in backend +#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] pub enum PyVersion { Py310, + #[default] Py311, Py312, Py313, - System, } impl PyVersion { + pub async fn from_instance_version() -> Self { + INSTANCE_PYTHON_VERSION + .read() + .await + .clone() + .and_then(|v| PyVersion::from_index(&v)) + .unwrap_or_else(|| { + let v = PyVersion::default(); + tracing::error!( + "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", + *INSTANCE_PYTHON_VERSION + ); + v + }) + } /// e.g.: `/tmp/windmill/cache/python_3xy` pub fn to_cache_dir(&self) -> String { use windmill_common::worker::ROOT_CACHE_DIR; @@ -130,7 +149,6 @@ impl PyVersion { Py311 => "3.11", Py312 => "3.12", Py313 => "3.13", - System => "sys", } } pub fn from_string_with_dots(value: &str) -> Option { @@ -140,7 +158,19 @@ impl PyVersion { "3.11" => Some(Py311), "3.12" => Some(Py312), "3.13" => Some(Py313), - "sys" => Some(System), + "latest_stable" => Some(PyVersion::default()), + _ => None, + } + } + // Check frontend for more context + pub fn from_index(value: &str) -> Option { + use PyVersion::*; + match value { + "0" => Some(PyVersion::default()), + "1" => Some(Py310), + "2" => Some(Py311), + "3" => Some(Py312), + "4" => Some(Py313), _ => None, } } @@ -149,7 +179,7 @@ impl PyVersion { Self::from_string_with_dots(line.replace("# py-", "").as_str()) } pub fn from_py_annotations(a: PythonAnnotations) -> Option { - let PythonAnnotations { py_sys, py310, py311, py312, py313, .. } = a; + let PythonAnnotations { py310, py311, py312, py313, py_latest_stable, .. } = a; use PyVersion::*; if py313 { Some(Py313) @@ -159,8 +189,8 @@ impl PyVersion { Some(Py311) } else if py310 { Some(Py310) - } else if py_sys { - Some(System) + } else if py_latest_stable { + Some(PyVersion::default()) } else { None } @@ -283,9 +313,6 @@ impl PyVersion { w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result> { - if matches!(self, Self::System) || *USE_SYSTEM_PYTHON { - return Ok(None); - } // lazy_static::lazy_static! { // static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); // } @@ -562,16 +589,12 @@ pub async fn uv_pip_compile( "--cache-dir", UV_CACHE_DIR, ]; - if !matches!(py_version, PyVersion::System) { - args.extend([ - "-p", - py_version.to_string_with_dot(), - "--python-preference", - "only-managed", - ]); - } else { - args.extend(["--python-preference", "only-system"]); - } + args.extend([ + "-p", + py_version.to_string_with_dot(), + "--python-preference", + "only-managed", + ]); if no_cache { args.extend(["--no-cache"]); } @@ -838,19 +861,11 @@ pub async fn handle_python_job( tracing::debug!("Finished deps postinstall stage"); } - append_logs( - &job.id, - &job.workspace_id, - "\n\n--- PYTHON CODE EXECUTION ---\n".to_string(), - db, - ) - .await; - if no_uv { append_logs( &job.id, &job.workspace_id, - format!("\n\n--- PYTHON 3.11 (Fallback) CODE EXECUTION ---\n",), + format!("\n\n--- SYSTEM PYTHON (Fallback) CODE EXECUTION ---\n",), db, ) .await; @@ -1402,18 +1417,7 @@ async fn handle_python_deps( .clone(); let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); - let instance_version = INSTANCE_PYTHON_VERSION - .read() - .await - .clone() - .and_then(|v| PyVersion::from_string_with_dots(&v)) - .unwrap_or_else(|| { - tracing::warn!( - "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to 3.11", - *INSTANCE_PYTHON_VERSION - ); - PyVersion::Py311 - }); + let instance_version = PyVersion::from_instance_version().await; let annotated_version = PyVersion::from_py_annotations(annotations); let mut is_deployed = true; @@ -2058,17 +2062,21 @@ pub async fn handle_python_reqs( let is_not_pro = !matches!(get_license_plan().await, LicensePlan::Pro); let total_time = std::time::Instant::now(); - let py_path = py_version - .get_python( - &job_dir, - job_id, - mem_peak, - db, - _worker_name, - w_id, - _occupancy_metrics, - ) - .await?; + let py_path = if no_uv_install { + None + } else { + py_version + .get_python( + &job_dir, + job_id, + mem_peak, + db, + _worker_name, + w_id, + _occupancy_metrics, + ) + .await? + }; let has_work = req_with_penv.len() > 0; for ((i, (req, venv_p)), mut kill_rx) in diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 76c9f4b0bddc0..23dd32e593ebb 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -106,7 +106,7 @@ use crate::{ js_eval::{eval_fetch_timeout, transpile_ts}, pg_executor::do_postgresql, php_executor::handle_php_job, - python_executor::handle_python_job, + python_executor::{handle_python_job, PyVersion}, result_processor::{process_result, start_background_processor}, rust_executor::handle_rust_job, worker_flow::{handle_flow, update_flow_status_in_progress}, @@ -258,13 +258,11 @@ pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); -pub const PYSYS_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_sys"); pub const TAR_PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_310"); pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"); pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312"); pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313"); -pub const TAR_PYSYS_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_sys"); pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_install"); @@ -762,6 +760,19 @@ pub async fn run_worker( let worker_dir = format!("{TMP_DIR}/{worker_name}"); tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); + if let Err(e) = PyVersion::from_instance_version() + .await + .get_python("", &Uuid::nil(), &mut 0, db, &worker_name, "", &mut None) + .await + { + tracing::error!( + worker = %worker_name, + hostname = %hostname, + worker_dir = %worker_dir, + "Cannot install/find Instance Python version to worker: {e}"// + ); + } + if let Some(ref netrc) = *NETRC { tracing::info!(worker = %worker_name, hostname = %hostname, "Writing netrc at {}/.netrc", HOME_ENV.as_str()); write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc"); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 9ee1941b09594..f0eb32227c609 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1584,9 +1584,6 @@ async fn python_dep( ) -> std::result::Result { create_dependencies_dir(job_dir).await; - let instance_version = - (INSTANCE_PYTHON_VERSION.read().await.clone()).unwrap_or("3.11".to_owned()); - /* Unlike `handle_python_deps` which we use for running scripts (deployed and drafts) This one used specifically for deploying scripts @@ -1597,15 +1594,8 @@ async fn python_dep( 2. Instance version 3. 311 */ - let final_version = PyVersion::from_py_annotations(annotations).unwrap_or( - PyVersion::from_string_with_dots(&instance_version).unwrap_or_else(|| { - tracing::error!( - "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to 3.11", - *INSTANCE_PYTHON_VERSION - ); - PyVersion::Py311 - }), - ); + let final_version = PyVersion::from_py_annotations(annotations) + .unwrap_or(PyVersion::from_instance_version().await); let req: std::result::Result = uv_pip_compile( job_id, diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 68ff3e1edf3ac..17c144aade4e1 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -26,6 +26,8 @@ import { createEventDispatcher } from 'svelte' import { fade } from 'svelte/transition' import { base } from '$lib/base' + import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' export let setting: Setting export let version: string @@ -775,8 +777,16 @@ bind:seconds={$values[setting.key]} /> + {:else if setting.fieldType == 'select'} +
+ + {#each ((setting.placeholder ?? 'parsing error').split(',')) as item, index} + + {/each} + +
{/if} - {#if hasError} {setting.error ?? ''} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 6af583fc82503..168ac2eedb72a 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -217,6 +217,18 @@ export const settings: Record = { ], 'Auth/OAuth': [], Registries: [ + { + label: 'Instance Python Version', + description: 'Default python version for newly deployed scripts', + key: 'instance_python_version', + fieldType: 'select', + // To change latest stable version: + // 1. Change placeholder in instanceSettings.ts + // 2. Change LATEST_STABLE_PY in dockerfile + // 3. Change #[default] annotation for PyVersion in backend + placeholder: 'latest stable (3.11),3.10,3.11,3.12,3.13', + storage: 'setting', + }, { label: 'Pip Index Url', description: 'Add private Pip registry', @@ -235,14 +247,6 @@ export const settings: Record = { storage: 'setting', ee_only: '' }, - { - label: 'Instance Python Version', - description: 'Default python version for newly deployed scripts', - key: 'instance_python_version', - fieldType: 'text', - placeholder: '3.10, 3.11 (Default), 3.12 or 3.13', - storage: 'setting', - }, { label: 'Npm Config Registry', description: 'Add private npm registry', From 02dbe8d69cdd28e1c56aafca9acef28f7545fb31 Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 13 Dec 2024 14:25:20 +0100 Subject: [PATCH 42/56] Load INSTANCE_PYTHON_VERSION on startup --- backend/src/main.rs | 17 +++++++++-------- backend/src/monitor.rs | 1 + .../src/lib/components/InstanceSetting.svelte | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 84afb8e3c5ee9..9e1c6c5c66b77 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -35,11 +35,12 @@ use windmill_common::{ CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, - LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, - OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, + KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, + NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, stats_ee::schedule_stats, @@ -68,9 +69,9 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, PYSYS_CACHE_DIR, - RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, - TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, TAR_PYSYS_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, + PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, + TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, + TAR_PY313_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 6a1312fb4621b..c1d36e9505226 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -188,6 +188,7 @@ pub async fn initial_load( reload_pip_index_url_setting(&db).await; reload_npm_config_registry_setting(&db).await; reload_bunfig_install_scopes_setting(&db).await; + reload_instance_python_version_setting(&db).await; } } diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 17c144aade4e1..2d5e17acb7c2e 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -782,7 +782,7 @@ {#each ((setting.placeholder ?? 'parsing error').split(',')) as item, index} - + {/each} From c1c1acc0e79deb2b6da39d295ab8b6c304d1ad60 Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 13 Dec 2024 19:16:49 +0100 Subject: [PATCH 43/56] Check for multiple annotations used --- .../windmill-parser-py-imports/src/lib.rs | 41 +++++++++++- .../windmill-parser-py-imports/tests/tests.rs | 3 + backend/windmill-common/src/worker.rs | 1 - .../windmill-worker/src/python_executor.rs | 64 +++++++++++-------- .../windmill-worker/src/worker_lockfiles.rs | 22 +++++-- .../src/lib/components/InstanceSetting.svelte | 33 +++++++--- .../src/lib/components/instanceSettings.ts | 61 ++++++++++++------ 7 files changed, 163 insertions(+), 62 deletions(-) diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index edd7221a0fbbf..94cf356d04544 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -21,7 +21,7 @@ use rustpython_parser::{ Parse, }; use sqlx::{Pool, Postgres}; -use windmill_common::error; +use windmill_common::{error, worker::PythonAnnotations}; const DEF_MAIN: &str = "def main("; @@ -178,7 +178,42 @@ pub async fn parse_python_imports( path: &str, db: &Pool, already_visited: &mut Vec, + annotated_pyv: &mut Option, ) -> error::Result> { + let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); + + // we pass only if there is none or only one annotation + + // Naive: + // 1. Check if there are multiple annotated version + // 2. If no, take one and compare with annotated version + // 3. We continue if same or replace none with new one + + // Optimized: + // 1. Iterate over all annotations compare each with annotated_pyv and replace on flight + // 2. If annotated_pyv is different version, throw and error + + // This way we make sure there is no multiple annotations for same script + // and we get detailed span on conflicting versions + + let mut check = |py_xyz, numeric| -> error::Result<()> { + if py_xyz { + if let Some(v) = annotated_pyv { + if *v != numeric { + return Err(error::Error::from(anyhow::anyhow!("No-no"))); + } + } else { + *annotated_pyv = Some(numeric); + } + } + Ok(()) + }; + + check(py310, 310)?; + check(py311, 311)?; + check(py312, 312)?; + check(py313, 313)?; + let find_requirements = code .lines() .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:")); @@ -225,11 +260,13 @@ pub async fn parse_python_imports( .fetch_optional(db) .await? .unwrap_or_else(|| "".to_string()); + if already_visited.contains(&rpath) { vec![] } else { already_visited.push(rpath.clone()); - parse_python_imports(&code, w_id, &rpath, db, already_visited).await? + parse_python_imports(&code, w_id, &rpath, db, already_visited, annotated_pyv) + .await? } } else { vec![replace_import(n.to_string())] diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index 436493e132743..a634f247dddce 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -25,6 +25,7 @@ def main(): "f/foo/bar", &db, &mut already_visited, + &mut None, ) .await?; // println!("{}", serde_json::to_string(&r)?); @@ -57,6 +58,7 @@ def main(): "f/foo/bar", &db, &mut already_visited, + &mut None, ) .await?; println!("{}", serde_json::to_string(&r)?); @@ -87,6 +89,7 @@ def main(): "f/foo/bar", &db, &mut already_visited, + &mut None, ) .await?; println!("{}", serde_json::to_string(&r)?); diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 4c4a65d011abf..60027b01f7bde 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -339,7 +339,6 @@ pub struct PythonAnnotations { pub no_uv_install: bool, pub no_uv_compile: bool, pub no_postinstall: bool, - pub py_latest_stable: bool, pub py310: bool, pub py311: bool, pub py312: bool, diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 9eaf30cb6ab43..e6858ac146e08 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -107,19 +107,18 @@ pub enum PyVersion { impl PyVersion { pub async fn from_instance_version() -> Self { - INSTANCE_PYTHON_VERSION - .read() - .await - .clone() - .and_then(|v| PyVersion::from_index(&v)) - .unwrap_or_else(|| { + match INSTANCE_PYTHON_VERSION.read().await.clone() { + Some(v) => PyVersion::from_string_with_dots(&v).unwrap_or_else(|| { let v = PyVersion::default(); tracing::error!( - "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", - *INSTANCE_PYTHON_VERSION - ); + "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", + *INSTANCE_PYTHON_VERSION + ); v - }) + }), + // Use latest stable + None => PyVersion::default(), + } } /// e.g.: `/tmp/windmill/cache/python_3xy` pub fn to_cache_dir(&self) -> String { @@ -158,19 +157,6 @@ impl PyVersion { "3.11" => Some(Py311), "3.12" => Some(Py312), "3.13" => Some(Py313), - "latest_stable" => Some(PyVersion::default()), - _ => None, - } - } - // Check frontend for more context - pub fn from_index(value: &str) -> Option { - use PyVersion::*; - match value { - "0" => Some(PyVersion::default()), - "1" => Some(Py310), - "2" => Some(Py311), - "3" => Some(Py312), - "4" => Some(Py313), _ => None, } } @@ -179,7 +165,7 @@ impl PyVersion { Self::from_string_with_dots(line.replace("# py-", "").as_str()) } pub fn from_py_annotations(a: PythonAnnotations) -> Option { - let PythonAnnotations { py310, py311, py312, py313, py_latest_stable, .. } = a; + let PythonAnnotations { py310, py311, py312, py313, .. } = a; use PyVersion::*; if py313 { Some(Py313) @@ -189,12 +175,29 @@ impl PyVersion { Some(Py311) } else if py310 { Some(Py310) - } else if py_latest_stable { - Some(PyVersion::default()) } else { None } } + pub fn from_numeric(n: u32) -> Option { + use PyVersion::*; + match n { + 310 => Some(Py310), + 311 => Some(Py311), + 312 => Some(Py312), + 313 => Some(Py313), + _ => None, + } + } + pub fn to_numeric(&self) -> u32 { + use PyVersion::*; + match self { + Py310 => 310, + Py311 => 311, + Py312 => 312, + Py313 => 313, + } + } pub async fn install_python( job_dir: &str, job_id: &Uuid, @@ -1418,8 +1421,9 @@ async fn handle_python_deps( let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); let instance_version = PyVersion::from_instance_version().await; - let annotated_version = PyVersion::from_py_annotations(annotations); + let mut annotated_version = PyVersion::from_py_annotations(annotations); let mut is_deployed = true; + let mut annotated_pyv = annotated_version.map(|v| v.to_numeric()); let requirements = match requirements_o { Some(r) => r, @@ -1433,6 +1437,7 @@ async fn handle_python_deps( script_path, db, &mut already_visited, + &mut annotated_pyv, ) .await? .join("\n"); @@ -1461,6 +1466,11 @@ async fn handle_python_deps( } } }; + + if let Some(pyv) = annotated_pyv { + annotated_version = annotated_version.or(PyVersion::from_numeric(pyv)); + } + /* For deployed scripts we want to find out version in following order: 1. Annotated version diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index f0eb32227c609..6126845ebe827 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -33,7 +33,6 @@ use crate::python_executor::{ USE_PIP_INSTALL, }; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; -use crate::INSTANCE_PYTHON_VERSION; use crate::{ bun_executor::gen_bun_lockfile, deno_executor::generate_deno_lock, @@ -1578,6 +1577,7 @@ async fn python_dep( w_id: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + annotated_pyv: Option, annotations: PythonAnnotations, no_uv_compile: bool, no_uv_install: bool, @@ -1593,9 +1593,17 @@ async fn python_dep( 1. Annotation version 2. Instance version 3. 311 + + Also it is worth noting, that if we receive annotated_pyv, + we just parse it instead. */ - let final_version = PyVersion::from_py_annotations(annotations) - .unwrap_or(PyVersion::from_instance_version().await); + + let annotated_version = if let Some(pyv) = annotated_pyv { + PyVersion::from_numeric(pyv) + } else { + PyVersion::from_py_annotations(annotations) + }; + let final_version = annotated_version.unwrap_or(PyVersion::from_instance_version().await); let req: std::result::Result = uv_pip_compile( job_id, @@ -1660,6 +1668,10 @@ async fn capture_dependency_job( ) -> error::Result { match job_language { ScriptLang::Python3 => { + let annotations = PythonAnnotations::parse(job_raw_code); + let mut annotated_pyv = + PyVersion::from_py_annotations(annotations).map(|v| v.to_numeric()); + let reqs = if raw_deps { job_raw_code.to_string() } else { @@ -1671,12 +1683,12 @@ async fn capture_dependency_job( script_path, &db, &mut already_visited, + &mut annotated_pyv, ) .await? .join("\n") }; - let annotations = PythonAnnotations::parse(job_raw_code); let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = annotations; if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL { @@ -1705,6 +1717,7 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), + annotated_pyv, annotations, no_uv_compile | no_uv, no_uv_install | no_uv, @@ -1746,6 +1759,7 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), + None, PythonAnnotations::default(), false, false, diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 2d5e17acb7c2e..b84c83c69ff2b 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -33,7 +33,6 @@ export let version: string export let values: Writable> export let loading = true - const dispatch = createEventDispatcher() let latestKeyRenewalAttempt: { @@ -50,6 +49,11 @@ return true } + console.log( + "val", + $values[setting.key] + ) + let licenseKeyChanged = false let renewing = false let opening = false @@ -121,6 +125,24 @@ EE only {#if setting.ee_only != ''}{setting.ee_only}{/if} {/if} + {#if setting.fieldType == 'select'} +
+ + + + {#each (setting.select_items ?? []) as item, index} + + {/each} + +
+ {:else} + {/if} {/if} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 168ac2eedb72a..33f721843b1a9 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -6,22 +6,29 @@ export interface Setting { ee_only?: string tooltip?: string key: string + // If value is not specified for first element, it will automatcally use undefined + select_items?: { + label: string, + tooltip?: string, + // If not specified, label will be used + value?: any, + }[], fieldType: - | 'text' - | 'number' - | 'boolean' - | 'password' - | 'select' - | 'textarea' - | 'seconds' - | 'email' - | 'license_key' - | 'object_store_config' - | 'critical_error_channels' - | 'slack_connect' - | 'smtp_connect' - | 'indexer_rates' - | 'otel' + | 'text' + | 'number' + | 'boolean' + | 'password' + | 'select' + | 'textarea' + | 'seconds' + | 'email' + | 'license_key' + | 'object_store_config' + | 'critical_error_channels' + | 'slack_connect' + | 'smtp_connect' + | 'indexer_rates' + | 'otel' storage: SettingStorage advancedToggle?: { label: string @@ -72,9 +79,9 @@ export const settings: Record = { isValid: (value: string | undefined) => value ? value?.startsWith('http') && - value.includes('://') && - !value?.endsWith('/') && - !value?.endsWith(' ') + value.includes('://') && + !value?.endsWith('/') && + !value?.endsWith(' ') : false }, { @@ -226,7 +233,23 @@ export const settings: Record = { // 1. Change placeholder in instanceSettings.ts // 2. Change LATEST_STABLE_PY in dockerfile // 3. Change #[default] annotation for PyVersion in backend - placeholder: 'latest stable (3.11),3.10,3.11,3.12,3.13', + placeholder: '3.10,3.11,3.12,3.13', + select_items: [{ + label: "Latest Stable", + tooltip: "python-3.11", + }, + { + label: "3.10", + }, + { + label: "3.11", + }, + { + label: "3.12", + }, + { + label: "3.13", + }], storage: 'setting', }, { From e91d934a623475403fa6bbb32191d3833752ba51 Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 13 Dec 2024 20:01:49 +0100 Subject: [PATCH 44/56] Fix Latest Stable button not pressed if selected --- backend/windmill-worker/src/python_executor.rs | 1 + frontend/src/lib/components/InstanceSetting.svelte | 13 ++++++------- frontend/src/lib/components/instanceSettings.ts | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index e6858ac146e08..1f655348d7af3 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -157,6 +157,7 @@ impl PyVersion { "3.11" => Some(Py311), "3.12" => Some(Py312), "3.13" => Some(Py313), + "default" => Some(PyVersion::default()), _ => None, } } diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index b84c83c69ff2b..4c1061f0b422c 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -35,6 +35,10 @@ export let loading = true const dispatch = createEventDispatcher() + if (setting.fieldType == 'select' && $values[setting.key] == undefined){ + $values[setting.key] = "default"; + } + let latestKeyRenewalAttempt: { result: string attempted_at: string @@ -49,11 +53,6 @@ return true } - console.log( - "val", - $values[setting.key] - ) - let licenseKeyChanged = false let renewing = false let opening = false @@ -137,8 +136,8 @@ {/if} - {#each (setting.select_items ?? []) as item, index} - + {#each (setting.select_items ?? []) as item } + {/each} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 33f721843b1a9..9fef598bfddaf 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -236,6 +236,7 @@ export const settings: Record = { placeholder: '3.10,3.11,3.12,3.13', select_items: [{ label: "Latest Stable", + value: "default", tooltip: "python-3.11", }, { From 4c22da0d40977a58f5efbd2708256e0ff96ce60c Mon Sep 17 00:00:00 2001 From: pyranota Date: Wed, 18 Dec 2024 23:14:29 +0300 Subject: [PATCH 45/56] Proper error handling for conflict on multiple annotations --- .../windmill-parser-py-imports/src/lib.rs | 46 ++++++++++++++++--- .../windmill-worker/src/python_executor.rs | 44 ++++++++++-------- 2 files changed, 65 insertions(+), 25 deletions(-) diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 94cf356d04544..634334e8b52b3 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -171,7 +171,6 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> return Ok(nimports); } -#[async_recursion] pub async fn parse_python_imports( code: &str, w_id: &str, @@ -179,6 +178,28 @@ pub async fn parse_python_imports( db: &Pool, already_visited: &mut Vec, annotated_pyv: &mut Option, +) -> error::Result> { + parse_python_imports_inner( + code, + w_id, + path, + db, + already_visited, + annotated_pyv, + &mut annotated_pyv.and_then(|_| Some(path.to_owned())), + ) + .await +} + +#[async_recursion] +async fn parse_python_imports_inner( + code: &str, + w_id: &str, + path: &str, + db: &Pool, + already_visited: &mut Vec, + annotated_pyv: &mut Option, + path_where_annotated_pyv: &mut Option, ) -> error::Result> { let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); @@ -196,15 +217,20 @@ pub async fn parse_python_imports( // This way we make sure there is no multiple annotations for same script // and we get detailed span on conflicting versions - let mut check = |py_xyz, numeric| -> error::Result<()> { - if py_xyz { + let mut check = |is_py_xyz, numeric| -> error::Result<()> { + if is_py_xyz { if let Some(v) = annotated_pyv { if *v != numeric { - return Err(error::Error::from(anyhow::anyhow!("No-no"))); + return Err(error::Error::from(anyhow::anyhow!( + "Annotated 2 or more different python versions: \n - py{v} at {}\n - py{numeric} at {path}\nIt is possible to use only one.", + path_where_annotated_pyv.clone().unwrap_or("Unknown".to_owned()) + ))); } } else { *annotated_pyv = Some(numeric); } + + *path_where_annotated_pyv = Some(path.to_owned()); } Ok(()) }; @@ -265,8 +291,16 @@ pub async fn parse_python_imports( vec![] } else { already_visited.push(rpath.clone()); - parse_python_imports(&code, w_id, &rpath, db, already_visited, annotated_pyv) - .await? + parse_python_imports_inner( + &code, + w_id, + &rpath, + db, + already_visited, + annotated_pyv, + path_where_annotated_pyv, + ) + .await? } } else { vec![replace_import(n.to_string())] diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 1f655348d7af3..14efc03f8b16d 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -210,6 +210,13 @@ impl PyVersion { occupancy_metrics: &mut Option<&mut OccupancyMetrics>, version: &str, ) -> error::Result<()> { + append_logs( + job_id, + w_id, + format!("\n\n--- INSTALLING PYTHON ({}) ---\n", version), + db, + ) + .await; // Create dirs for newly installed python // If we dont do this, NSJAIL will not be able to mount cache // For the default version directory created during startup (main.rs) @@ -857,6 +864,24 @@ pub async fn handle_python_job( let PythonAnnotations { no_uv, no_postinstall, .. } = PythonAnnotations::parse(inner_content); tracing::debug!("Finished handling python dependencies"); + let python_path = if no_uv { + PYTHON_PATH.clone() + } else if let Some(python_path) = py_version + .get_python( + job_dir, + &job.id, + mem_peak, + db, + worker_name, + &job.workspace_id, + &mut Some(occupancy_metrics), + ) + .await? + { + python_path + } else { + PYTHON_PATH.clone() + }; if !no_postinstall { if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, db).await { @@ -1058,25 +1083,6 @@ mount {{ job.id ); - let python_path = if no_uv { - PYTHON_PATH.clone() - } else if let Some(python_path) = py_version - .get_python( - job_dir, - &job.id, - mem_peak, - db, - worker_name, - &job.workspace_id, - &mut Some(occupancy_metrics), - ) - .await? - { - python_path - } else { - PYTHON_PATH.clone() - }; - let child = if !*DISABLE_NSJAIL { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd From d41b70c2eda3f8a32b174b52b0208177037ebd1b Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 19 Dec 2024 14:44:16 +0300 Subject: [PATCH 46/56] Fix merge conflicts --- .../windmill-parser-py-imports/src/lib.rs | 14 +-- backend/src/main.rs | 10 +- backend/src/monitor.rs | 12 +- .../windmill-worker/src/python_executor.rs | 105 +++++++++--------- backend/windmill-worker/src/worker.rs | 4 +- .../windmill-worker/src/worker_lockfiles.rs | 96 ++++++++-------- 6 files changed, 121 insertions(+), 120 deletions(-) diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 634334e8b52b3..d0a5c9622b3a5 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -177,7 +177,7 @@ pub async fn parse_python_imports( path: &str, db: &Pool, already_visited: &mut Vec, - annotated_pyv: &mut Option, + annotated_pyv_numeric: &mut Option, ) -> error::Result> { parse_python_imports_inner( code, @@ -185,8 +185,8 @@ pub async fn parse_python_imports( path, db, already_visited, - annotated_pyv, - &mut annotated_pyv.and_then(|_| Some(path.to_owned())), + annotated_pyv_numeric, + &mut annotated_pyv_numeric.and_then(|_| Some(path.to_owned())), ) .await } @@ -198,7 +198,7 @@ async fn parse_python_imports_inner( path: &str, db: &Pool, already_visited: &mut Vec, - annotated_pyv: &mut Option, + annotated_pyv_numeric: &mut Option, path_where_annotated_pyv: &mut Option, ) -> error::Result> { let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); @@ -219,7 +219,7 @@ async fn parse_python_imports_inner( let mut check = |is_py_xyz, numeric| -> error::Result<()> { if is_py_xyz { - if let Some(v) = annotated_pyv { + if let Some(v) = annotated_pyv_numeric { if *v != numeric { return Err(error::Error::from(anyhow::anyhow!( "Annotated 2 or more different python versions: \n - py{v} at {}\n - py{numeric} at {path}\nIt is possible to use only one.", @@ -227,7 +227,7 @@ async fn parse_python_imports_inner( ))); } } else { - *annotated_pyv = Some(numeric); + *annotated_pyv_numeric = Some(numeric); } *path_where_annotated_pyv = Some(path.to_owned()); @@ -297,7 +297,7 @@ async fn parse_python_imports_inner( &rpath, db, already_visited, - annotated_pyv, + annotated_pyv_numeric, path_where_annotated_pyv, ) .await? diff --git a/backend/src/main.rs b/backend/src/main.rs index 9d5b10d400356..31c34e36c7b83 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -9,8 +9,9 @@ use anyhow::Context; use monitor::{ load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config, - reload_nuget_config_setting, reload_timeout_wait_result_setting, - send_current_log_file_to_object_store, send_logs_to_object_store, + reload_instance_python_version_setting, reload_nuget_config_setting, + reload_timeout_wait_result_setting, send_current_log_file_to_object_store, + send_logs_to_object_store, }; use rand::Rng; use sqlx::{postgres::PgListener, Pool, Postgres}; @@ -69,8 +70,9 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, - POWERSHELL_CACHE_DIR, PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR, - + POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, + RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, + TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index eebbc2f957f16..e556ae51059cb 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -41,10 +41,10 @@ use windmill_common::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, - JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, + NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -68,8 +68,8 @@ use windmill_common::{ use windmill_queue::cancel_job; use windmill_worker::{ create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, - SendResult, BUNFIG_INSTALL_SCOPES, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, - NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY, + SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, + NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY, }; #[cfg(feature = "parquet")] diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b9530129d2cad..be9847e1a13e4 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -157,13 +157,25 @@ impl PyVersion { "3.11" => Some(Py311), "3.12" => Some(Py312), "3.13" => Some(Py313), - "default" => Some(PyVersion::default()), _ => None, } } - /// e.g.: `# py-3.xy` -> `PyVersion::Py3XY` + pub fn from_string_no_dots(value: &str) -> Option { + use PyVersion::*; + match value { + "310" => Some(Py310), + "311" => Some(Py311), + "312" => Some(Py312), + "313" => Some(Py313), + _ => { + tracing::warn!("Cannot convert string (\"{value}\") to PyVersion"); + None + } + } + } + /// e.g.: `# py3xy` -> `PyVersion::Py3XY` pub fn parse_version(line: &str) -> Option { - Self::from_string_with_dots(line.replace("# py-", "").as_str()) + Self::from_string_no_dots(line.replace(" ", "").replace("#py", "").as_str()) } pub fn from_py_annotations(a: PythonAnnotations) -> Option { let PythonAnnotations { py310, py311, py312, py313, .. } = a; @@ -472,7 +484,7 @@ pub async fn uv_pip_compile( // Include python version to requirements.in // We need it because same hash based on requirements.in can get calculated even for different python versions // To prevent from overwriting same requirements.in but with different python versions, we include version to hash - let requirements = format!("# py-{}\n{}", py_version.to_string_with_dot(), requirements); + let requirements = format!("# py{}\n{}", py_version.to_string_no_dot(), requirements); #[cfg(feature = "enterprise")] let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?; @@ -681,8 +693,8 @@ pub async fn uv_pip_compile( let mut req_content = "".to_string(); file.read_to_string(&mut req_content).await?; let lockfile = format!( - "# py-{}\n{}", - py_version.to_string_with_dot(), + "# py{}\n{}", + py_version.to_string_no_dot(), req_content .lines() .filter(|x| !x.trim_start().starts_with('#')) @@ -1426,17 +1438,15 @@ async fn handle_python_deps( .unwrap_or_else(|| vec![]) .clone(); - let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); - let instance_version = PyVersion::from_instance_version().await; - let mut annotated_version = PyVersion::from_py_annotations(annotations); - let mut is_deployed = true; - let mut annotated_pyv = annotated_version.map(|v| v.to_numeric()); - let mut requirements; + let mut annotated_pyv = None; + let mut annotated_pyv_numeric = None; + let is_deployed = requirements_o.is_some(); + let instance_pyv = PyVersion::from_instance_version().await; + let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); let requirements = match requirements_o { Some(r) => r, None => { - is_deployed = false; let mut already_visited = vec![]; requirements = windmill_parser_py_imports::parse_python_imports( @@ -1445,10 +1455,13 @@ async fn handle_python_deps( script_path, db, &mut already_visited, - &mut annotated_pyv, + &mut annotated_pyv_numeric, ) .await? .join("\n"); + + annotated_pyv = annotated_pyv_numeric.and_then(|v| PyVersion::from_numeric(v)); + if !requirements.is_empty() { requirements = uv_pip_compile( job_id, @@ -1460,7 +1473,7 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - annotated_version.unwrap_or(instance_version), + annotated_pyv.unwrap_or(instance_pyv), annotations.no_uv || annotations.no_uv_compile, annotations.no_cache, ) @@ -1473,22 +1486,6 @@ async fn handle_python_deps( } }; - if let Some(pyv) = annotated_pyv { - annotated_version = annotated_version.or(PyVersion::from_numeric(pyv)); - } - - /* - For deployed scripts we want to find out version in following order: - 1. Annotated version - 2. Assigned version (written in lockfile) - 3. 3.11 - - For Previews: - 1. Annotated version - 2. Instance version - 3. 3.11 - */ - let requirements_lines: Vec<&str> = if requirements.len() > 0 { requirements .split("\n") @@ -1498,28 +1495,34 @@ async fn handle_python_deps( vec![] }; - let final_version = annotated_version.unwrap_or_else(|| { - if is_deployed { - // If script is deployed we can try to parse first line to get assigned version - if let Some(v) = requirements_lines - .get(0) - .and_then(|line| PyVersion::parse_version(line)) - { - // We have assigned version, we should use it - v - } else { - // If there is no assigned version in lockfile we automatically fallback to 3.11 - // In this case we have dependencies, but no associated python version - // This is the case for old deployed scripts - PyVersion::Py311 - } + /* + For deployed scripts we want to find out version in following order: + 1. Assigned version (written in lockfile) + 2. 3.11 + + For Previews: + 1. Annotated version + 2. Instance version + 3. Latest Stable + */ + let final_version = if is_deployed { + // If script is deployed we can try to parse first line to get assigned version + if let Some(v) = requirements_lines + .get(0) + .and_then(|line| PyVersion::parse_version(line)) + { + // We have valid assigned version, we use it + v } else { - // This is not deployed script, meaning we test run it (Preview) - // In this case we can say that desired version is `instance_version` - instance_version + // If there is no assigned version in lockfile we automatically fallback to 3.11 + // In this case we have dependencies, but no associated python version + // This is the case for old deployed scripts + PyVersion::Py311 } - }); - + } else { + // This is not deployed script, meaning we test run it (Preview) + annotated_pyv.unwrap_or(instance_pyv) + }; // If len > 0 it means there is atleast one dependency or assigned python version if requirements.len() > 0 { let mut venv_path = handle_python_reqs( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 21bfcf10a699a..b13c53d82f108 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -105,8 +105,6 @@ use crate::{ job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, pg_executor::do_postgresql, - php_executor::handle_php_job, - python_executor::{handle_python_job, PyVersion}, result_processor::{process_result, start_background_processor}, worker_flow::{handle_flow, update_flow_status_in_progress}, worker_lockfiles::{ @@ -121,7 +119,7 @@ use crate::rust_executor::handle_rust_job; use crate::php_executor::handle_php_job; #[cfg(feature = "python")] -use crate::python_executor::handle_python_job; +use crate::python_executor::{handle_python_job, PyVersion}; #[cfg(feature = "python")] use crate::ansible_executor::handle_ansible_job; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index dc2d193552bbc..97d501c4b8b72 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1595,7 +1595,7 @@ async fn python_dep( w_id: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - annotated_pyv: Option, + annotated_pyv_numeric: Option, annotations: PythonAnnotations, no_uv_compile: bool, no_uv_install: bool, @@ -1610,18 +1610,12 @@ async fn python_dep( 1. Annotation version 2. Instance version - 3. 311 - - Also it is worth noting, that if we receive annotated_pyv, - we just parse it instead. + 3. Latest Stable */ - let annotated_version = if let Some(pyv) = annotated_pyv { - PyVersion::from_numeric(pyv) - } else { - PyVersion::from_py_annotations(annotations) - }; - let final_version = annotated_version.unwrap_or(PyVersion::from_instance_version().await); + let final_version = annotated_pyv_numeric + .and_then(|pyv| PyVersion::from_numeric(pyv)) + .unwrap_or(PyVersion::from_instance_version().await); let req: std::result::Result = uv_pip_compile( job_id, @@ -1692,26 +1686,28 @@ async fn capture_dependency_job( )); #[cfg(feature = "python")] { - let annotations = PythonAnnotations::parse(job_raw_code); - let mut annotated_pyv = - PyVersion::from_py_annotations(annotations).map(|v| v.to_numeric()); - - let reqs = if raw_deps { - job_raw_code.to_string() - } else { - let mut already_visited = vec![]; - - windmill_parser_py_imports::parse_python_imports( - job_raw_code, - &w_id, - script_path, - &db, - &mut already_visited, - &mut annotated_pyv, - ) - .await? - .join("\n") - }; + let anns = PythonAnnotations::parse(job_raw_code); + let mut annotated_pyv_numeric = None; + + let reqs = if raw_deps { + annotated_pyv_numeric = + PyVersion::from_py_annotations(anns).map(|v| v.to_numeric()); + job_raw_code.to_string() + } else { + let mut already_visited = vec![]; + + windmill_parser_py_imports::parse_python_imports( + job_raw_code, + &w_id, + script_path, + &db, + &mut already_visited, + &mut annotated_pyv_numeric, + ) + .await? + .join("\n") + }; + let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = anns; if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL { if let Err(e) = sqlx::query!( r#" @@ -1727,24 +1723,24 @@ async fn capture_dependency_job( } } - python_dep( - reqs, - job_id, - mem_peak, - canceled_by, - job_dir, - db, - worker_name, - w_id, - worker_dir, - &mut Some(occupancy_metrics), - annotated_pyv, - annotations, - no_uv_compile | no_uv, - no_uv_install | no_uv, - ) - .await - } + python_dep( + reqs, + job_id, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + w_id, + worker_dir, + &mut Some(occupancy_metrics), + annotated_pyv_numeric, + anns, + no_uv_compile | no_uv, + no_uv_install | no_uv, + ) + .await + } } ScriptLang::Ansible => { #[cfg(not(feature = "python"))] @@ -1788,6 +1784,8 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), + None, + PythonAnnotations::default(), false, false, ) From 05ba8bfd852adb620c102b39a6e9587e3e6de8e6 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 19 Dec 2024 14:48:16 +0300 Subject: [PATCH 47/56] Preinstall 3.11 and Latest Stable --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f11c3518c3c5c..66067b993e6a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -170,7 +170,8 @@ ENV GO_PATH=/usr/local/go/bin/go # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -# Preinstall default python version +# Preinstall python runtimes +RUN uv python install 3.11.10 RUN uv python install $LATEST_STABLE_PY RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - From c4ec30409cd8948f1d066eb53a0dd69b9435ad08 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 19 Dec 2024 15:14:34 +0300 Subject: [PATCH 48/56] Preinstall latest stable in non-blocking manner --- .../windmill-worker/src/python_executor.rs | 40 ++++++++----------- backend/windmill-worker/src/worker.rs | 29 +++++++++----- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index be9847e1a13e4..2e3a45e49677a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -157,7 +157,12 @@ impl PyVersion { "3.11" => Some(Py311), "3.12" => Some(Py312), "3.13" => Some(Py313), - _ => None, + _ => { + tracing::warn!( + "Cannot convert string (\"{value}\") to PyVersion\nExpected format x.yz" + ); + None + } } } pub fn from_string_no_dots(value: &str) -> Option { @@ -168,7 +173,9 @@ impl PyVersion { "312" => Some(Py312), "313" => Some(Py313), _ => { - tracing::warn!("Cannot convert string (\"{value}\") to PyVersion"); + tracing::warn!( + "Cannot convert string (\"{value}\") to PyVersion\nExpected format xyz" + ); None } } @@ -212,7 +219,6 @@ impl PyVersion { } } pub async fn install_python( - job_dir: &str, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -225,7 +231,7 @@ impl PyVersion { append_logs( job_id, w_id, - format!("\n\n--- INSTALLING PYTHON ({}) ---\n", version), + format!("\n\nINSTALLING PYTHON ({})", version), db, ) .await; @@ -248,7 +254,6 @@ impl PyVersion { // let v_with_dot = self.to_string_with_dot(); let mut child_cmd = Command::new(UV_PATH.as_str()); child_cmd - .current_dir(job_dir) .args([ "python", "install", @@ -280,7 +285,6 @@ impl PyVersion { .await } async fn get_python_inner( - job_dir: &str, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -290,13 +294,12 @@ impl PyVersion { occupancy_metrics: &mut Option<&mut OccupancyMetrics>, version: &str, ) -> error::Result> { - let py_path = Self::find_python(job_dir, version).await; + let py_path = Self::find_python(version).await; - // Python is not installed + // Runtime is not installed if py_path.is_err() { // Install it if let Err(err) = Self::install_python( - job_dir, job_id, mem_peak, db, @@ -311,7 +314,7 @@ impl PyVersion { return Err(err); } else { // Try to find one more time - let py_path = Self::find_python(job_dir, version).await; + let py_path = Self::find_python(version).await; if let Err(err) = py_path { tracing::error!("Cannot find python version {err}"); @@ -327,7 +330,6 @@ impl PyVersion { } pub async fn get_python( &self, - job_dir: &str, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -341,7 +343,6 @@ impl PyVersion { // } let res = Self::get_python_inner( - job_dir, job_id, mem_peak, db, @@ -360,12 +361,12 @@ impl PyVersion { } res } - async fn find_python(job_dir: &str, version: &str) -> error::Result> { + async fn find_python(version: &str) -> error::Result> { // let mut logs = String::new(); // let v_with_dot = self.to_string_with_dot(); let mut child_cmd = Command::new(UV_PATH.as_str()); let output = child_cmd - .current_dir(job_dir) + // .current_dir(job_dir) .args([ "python", "find", @@ -880,7 +881,6 @@ pub async fn handle_python_job( PYTHON_PATH.clone() } else if let Some(python_path) = py_version .get_python( - job_dir, &job.id, mem_peak, db, @@ -2085,15 +2085,7 @@ pub async fn handle_python_reqs( None } else { py_version - .get_python( - &job_dir, - job_id, - mem_peak, - db, - _worker_name, - w_id, - _occupancy_metrics, - ) + .get_python(job_id, mem_peak, db, _worker_name, w_id, _occupancy_metrics) .await? }; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b13c53d82f108..73ecb05ef27a3 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -79,6 +79,7 @@ use tokio::fs::symlink; use tokio::fs::symlink_file as symlink; use tokio::{ + spawn, sync::{ mpsc::{self, Sender}, RwLock, @@ -777,17 +778,27 @@ pub async fn run_worker( let worker_dir = format!("{TMP_DIR}/{worker_name}"); tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); - if let Err(e) = PyVersion::from_instance_version() - .await - .get_python("", &Uuid::nil(), &mut 0, db, &worker_name, "", &mut None) - .await { - tracing::error!( - worker = %worker_name, - hostname = %hostname, - worker_dir = %worker_dir, - "Cannot install/find Instance Python version to worker: {e}"// + let (db, worker_name, hostname, worker_dir) = ( + db.clone(), + worker_name.clone(), + hostname.to_owned(), + worker_dir.clone(), ); + spawn(async move { + if let Err(e) = PyVersion::from_instance_version() + .await + .get_python(&Uuid::nil(), &mut 0, &db, &worker_name, "", &mut None) + .await + { + tracing::error!( + worker = %worker_name, + hostname = %hostname, + worker_dir = %worker_dir, + "Cannot preinstall or find Instance Python version to worker: {e}"// + ); + } + }); } if let Some(ref netrc) = *NETRC { From 5a5ea8b6f1d943a0de5a069fb824ad02e0a62c6b Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 19 Dec 2024 15:20:51 +0300 Subject: [PATCH 49/56] Fix Warning --- backend/src/main.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 31c34e36c7b83..ba50805b81bf6 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -36,12 +36,12 @@ use windmill_common::{ CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, - LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, - NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, - TIMEOUT_WAIT_RESULT_SETTING, + INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, + KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, + NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, + PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, stats_ee::schedule_stats, From 611355ee640684ad1a4d8658888ddafe22f1f99a Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 19 Dec 2024 15:26:17 +0300 Subject: [PATCH 50/56] Gate preinstall logic behind "python" feature --- backend/windmill-worker/src/worker.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 73ecb05ef27a3..734c3aab5139f 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -79,7 +79,6 @@ use tokio::fs::symlink; use tokio::fs::symlink_file as symlink; use tokio::{ - spawn, sync::{ mpsc::{self, Sender}, RwLock, @@ -778,6 +777,7 @@ pub async fn run_worker( let worker_dir = format!("{TMP_DIR}/{worker_name}"); tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); + #[cfg(feature = "python")] { let (db, worker_name, hostname, worker_dir) = ( db.clone(), @@ -785,7 +785,7 @@ pub async fn run_worker( hostname.to_owned(), worker_dir.clone(), ); - spawn(async move { + tokio::spawn(async move { if let Err(e) = PyVersion::from_instance_version() .await .get_python(&Uuid::nil(), &mut 0, &db, &worker_name, "", &mut None) From dec2facdd7397ea764d7077551d9b68d9d7b8f98 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 19 Dec 2024 17:10:00 +0300 Subject: [PATCH 51/56] Handle raw_deps properly --- backend/windmill-worker/src/worker_lockfiles.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 97d501c4b8b72..4c55f0de581c2 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1690,6 +1690,10 @@ async fn capture_dependency_job( let mut annotated_pyv_numeric = None; let reqs = if raw_deps { + // `wmill script generate-metadata` + // should also respect annotated pyversion + // can be annotated in script itself + // or in requirements.txt if present annotated_pyv_numeric = PyVersion::from_py_annotations(anns).map(|v| v.to_numeric()); job_raw_code.to_string() From a0e452f84a164ac4842bb25a03bc2fb6428dc109 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 19 Dec 2024 19:55:59 +0300 Subject: [PATCH 52/56] Make it work with nsjail --- Dockerfile | 2 +- .../nsjail/download.py.config.proto | 9 +- .../nsjail/run.python3.config.proto | 8 + .../windmill-worker/src/python_executor.rs | 164 ++++++++---------- backend/windmill-worker/src/worker.rs | 2 +- 5 files changed, 83 insertions(+), 102 deletions(-) diff --git a/Dockerfile b/Dockerfile index 69af553f62360..b27e0a28b331f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,7 +101,7 @@ ARG WITH_GIT=true # 2. Change LATEST_STABLE_PY in dockerfile # 3. Change #[default] annotation for PyVersion in backend ARG LATEST_STABLE_PY=3.11.10 -ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_install +ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_runtime ENV UV_PYTHON_PREFERENCE=only-managed RUN pip install --upgrade pip==24.2 diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index 78aef4fca6c33..b910e1c690856 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -20,6 +20,7 @@ clone_newuser: {CLONE_NEWUSER} keep_caps: true keep_env: true +mount_proc: true mount { src: "/bin" @@ -58,14 +59,6 @@ mount { is_bind: true rw: true } -# We need it for uv -# TODO: Dont expose /proc here, it is not safe -mount { - src: "/proc" - dst: "/proc" - is_bind: true - rw: true -} mount { dst: "/tmp" diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 06ced731c35de..386d7e5ce1ffa 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -16,6 +16,7 @@ clone_newuser: {CLONE_NEWUSER} keep_caps: false keep_env: true +mount_proc: true mount { src: "/bin" @@ -110,6 +111,13 @@ mount { is_bind: true } +#TODO: Make dynamic +mount { + src: "/tmp/windmill/cache/py_runtime" + dst: "/tmp/windmill/cache/py_runtime" + is_bind: true +} + mount { src: "/dev/urandom" dst: "/dev/urandom" diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 2e3a45e49677a..bdc0b5d8f0b48 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -218,7 +218,8 @@ impl PyVersion { Py313 => 313, } } - pub async fn install_python( + pub async fn get_python( + &self, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -226,65 +227,26 @@ impl PyVersion { worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - version: &str, - ) -> error::Result<()> { - append_logs( - job_id, - w_id, - format!("\n\nINSTALLING PYTHON ({})", version), - db, - ) - .await; - // Create dirs for newly installed python - // If we dont do this, NSJAIL will not be able to mount cache - // For the default version directory created during startup (main.rs) - DirBuilder::new() - .recursive(true) - .create( - PyVersion::from_string_with_dots(version) - .ok_or(error::Error::BadRequest( - "Invalid python version".to_owned(), - ))? - .to_cache_dir(), - ) - .await - .expect("could not create initial worker dir"); - - let logs = String::new(); - // let v_with_dot = self.to_string_with_dot(); - let mut child_cmd = Command::new(UV_PATH.as_str()); - child_cmd - .args([ - "python", - "install", - version, - "--python-preference=only-managed", - ]) - // TODO: Do we need these? - .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + ) -> error::Result> { + // lazy_static::lazy_static! { + // static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); + // } - let child_process = start_child_process(child_cmd, "uv").await?; + let res = self + .get_python_inner(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .await; - append_logs(&job_id, &w_id, logs, db).await; - handle_child( - job_id, - db, - mem_peak, - &mut None, - child_process, - false, - worker_name, - &w_id, - "uv", - None, - false, - occupancy_metrics, - ) - .await + if let Err(ref e) = res { + tracing::error!( + "worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n + Error while getting python from uv, falling back to system python: {e:?}" + ); + } + res } + #[async_recursion::async_recursion] async fn get_python_inner( + self, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -292,29 +254,21 @@ impl PyVersion { worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - version: &str, ) -> error::Result> { - let py_path = Self::find_python(version).await; + let py_path = self.find_python().await; // Runtime is not installed if py_path.is_err() { // Install it - if let Err(err) = Self::install_python( - job_id, - mem_peak, - db, - worker_name, - w_id, - occupancy_metrics, - version, - ) - .await + if let Err(err) = self + .get_python_inner(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .await { tracing::error!("Cannot install python: {err}"); return Err(err); } else { // Try to find one more time - let py_path = Self::find_python(version).await; + let py_path = self.find_python().await; if let Err(err) = py_path { tracing::error!("Cannot find python version {err}"); @@ -328,8 +282,8 @@ impl PyVersion { py_path } } - pub async fn get_python( - &self, + async fn install_python( + self, job_id: &Uuid, mem_peak: &mut i32, // canceled_by: &mut Option, @@ -337,31 +291,48 @@ impl PyVersion { worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - ) -> error::Result> { - // lazy_static::lazy_static! { - // static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); - // } + ) -> error::Result<()> { + let v = self.to_string_with_dot(); + append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), db).await; + // Create dirs for newly installed python + // If we dont do this, NSJAIL will not be able to mount cache + // For the default version directory created during startup (main.rs) + DirBuilder::new() + .recursive(true) + .create(self.to_cache_dir()) + .await + .expect("could not create initial worker dir"); + + let logs = String::new(); + // let v_with_dot = self.to_string_with_dot(); + let mut child_cmd = Command::new(UV_PATH.as_str()); + child_cmd + .args(["python", "install", v, "--python-preference=only-managed"]) + // TODO: Do we need these? + .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let child_process = start_child_process(child_cmd, "uv").await?; - let res = Self::get_python_inner( + append_logs(&job_id, &w_id, logs, db).await; + handle_child( job_id, - mem_peak, db, + mem_peak, + &mut None, + child_process, + false, worker_name, - w_id, + &w_id, + "uv", + None, + false, occupancy_metrics, - self.to_string_with_dot(), ) - .await; - - if let Err(ref e) = res { - tracing::error!( - "worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n - Error while getting python from uv, falling back to system python: {e:?}" - ); - } - res + .await } - async fn find_python(version: &str) -> error::Result> { + async fn find_python(self) -> error::Result> { // let mut logs = String::new(); // let v_with_dot = self.to_string_with_dot(); let mut child_cmd = Command::new(UV_PATH.as_str()); @@ -370,7 +341,7 @@ impl PyVersion { .args([ "python", "find", - version, + self.to_string_with_dot(), "--python-preference=only-managed", ]) .envs([ @@ -595,6 +566,11 @@ pub async fn uv_pip_compile( .await .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; } else { + // Make sure we have python runtime installed + py_version + .install_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .await?; + let mut args = vec![ "pip", "compile", @@ -613,12 +589,14 @@ pub async fn uv_pip_compile( "--cache-dir", UV_CACHE_DIR, ]; + args.extend([ "-p", - py_version.to_string_with_dot(), + &py_version.to_string_with_dot(), "--python-preference", "only-managed", ]); + if no_cache { args.extend(["--no-cache"]); } @@ -660,6 +638,7 @@ pub async fn uv_pip_compile( .env_clear() .env("HOME", HOME_ENV.to_string()) .env("PATH", PATH_ENV.to_string()) + .env("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR.to_string()) .args(&args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1076,6 +1055,7 @@ mount {{ "run.config.proto", &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT .replace("{JOB_DIR}", job_dir) + // .replace("{PY_RUNTIME_DIR}", &python_path) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str()) @@ -1474,8 +1454,8 @@ async fn handle_python_deps( w_id, occupancy_metrics, annotated_pyv.unwrap_or(instance_pyv), - annotations.no_uv || annotations.no_uv_compile, annotations.no_cache, + annotations.no_uv || annotations.no_uv_compile, ) .await .map_err(|e| { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 734c3aab5139f..bc3426c07275c 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -278,7 +278,7 @@ pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312" pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313"); pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); -pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_install"); +pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_runtime"); pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); From 770ff5a3589243f8e1839f84a05f2b00ecbaeff9 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Thu, 19 Dec 2024 21:48:07 +0300 Subject: [PATCH 53/56] Revert docker-image.yml --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 5e31dd0fa8626..25266fa4185f0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -8,7 +8,7 @@ env: name: Build windmill:main on: push: - branches: [main, multipython] + branches: [main] tags: ["*"] pull_request: types: [opened, synchronize, reopened] From ce34e00708d9a04325bd3024c1b675658d1c3532 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Thu, 19 Dec 2024 21:48:35 +0300 Subject: [PATCH 54/56] Revert Dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b27e0a28b331f..ba7b49474b91f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,3 @@ -# trigger CI #2 | Dont forget to remove ARG DEBIAN_IMAGE=debian:bookworm-slim ARG RUST_IMAGE=rust:1.82-slim-bookworm ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm From 3666694ef4e0bfbdfc7cd5ecd9e13460eef7c820 Mon Sep 17 00:00:00 2001 From: pyranota Date: Thu, 19 Dec 2024 21:51:19 +0300 Subject: [PATCH 55/56] Cleanup + Fixing --- .../nsjail/run.python3.config.proto | 5 ++--- backend/windmill-worker/src/python_executor.rs | 16 +++++----------- backend/windmill-worker/src/worker.rs | 11 +++++++++++ 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 386d7e5ce1ffa..a9d61dc8b13c8 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -111,10 +111,9 @@ mount { is_bind: true } -#TODO: Make dynamic mount { - src: "/tmp/windmill/cache/py_runtime" - dst: "/tmp/windmill/cache/py_runtime" + src: "{PY_INSTALL_DIR}" + dst: "{PY_INSTALL_DIR}" is_bind: true } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index bdc0b5d8f0b48..6f308a5137621 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -129,13 +129,6 @@ impl PyVersion { pub fn to_cache_dir_top_level(&self) -> String { format!("python_{}", self.to_string_no_dot()) } - /// e.g.: `(to_cache_dir(), to_cache_dir_top_level())` - #[cfg(all(feature = "enterprise", feature = "parquet"))] - pub fn to_cache_dir_tuple(&self) -> (String, String) { - use windmill_common::worker::ROOT_CACHE_DIR; - let top_level = self.to_cache_dir_top_level(); - (format!("{ROOT_CACHE_DIR}python_{}", &top_level), top_level) - } /// e.g.: `3xy` pub fn to_string_no_dot(&self) -> String { self.to_string_with_dot().replace('.', "") @@ -157,6 +150,7 @@ impl PyVersion { "3.11" => Some(Py311), "3.12" => Some(Py312), "3.13" => Some(Py313), + "default" => Some(PyVersion::default()), _ => { tracing::warn!( "Cannot convert string (\"{value}\") to PyVersion\nExpected format x.yz" @@ -172,6 +166,7 @@ impl PyVersion { "311" => Some(Py311), "312" => Some(Py312), "313" => Some(Py313), + "default" => Some(PyVersion::default()), _ => { tracing::warn!( "Cannot convert string (\"{value}\") to PyVersion\nExpected format xyz" @@ -244,7 +239,6 @@ impl PyVersion { } res } - #[async_recursion::async_recursion] async fn get_python_inner( self, job_id: &Uuid, @@ -261,7 +255,7 @@ impl PyVersion { if py_path.is_err() { // Install it if let Err(err) = self - .get_python_inner(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .install_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) .await { tracing::error!("Cannot install python: {err}"); @@ -568,7 +562,7 @@ pub async fn uv_pip_compile( } else { // Make sure we have python runtime installed py_version - .install_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .get_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) .await?; let mut args = vec![ @@ -1055,7 +1049,7 @@ mount {{ "run.config.proto", &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT .replace("{JOB_DIR}", job_dir) - // .replace("{PY_RUNTIME_DIR}", &python_path) + .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index bc3426c07275c..e53581437a578 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -798,6 +798,17 @@ pub async fn run_worker( "Cannot preinstall or find Instance Python version to worker: {e}"// ); } + if let Err(e) = PyVersion::Py311 + .get_python(&Uuid::nil(), &mut 0, &db, &worker_name, "", &mut None) + .await + { + tracing::error!( + worker = %worker_name, + hostname = %hostname, + worker_dir = %worker_dir, + "Cannot preinstall or find default 311 version to worker: {e}"// + ); + } }); } From ad84b244734493604a0224ad0a540abeb4fdf049 Mon Sep 17 00:00:00 2001 From: pyranota Date: Fri, 20 Dec 2024 17:13:44 +0300 Subject: [PATCH 56/56] Add windows support --- .../windmill-worker/src/python_executor.rs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 6f308a5137621..1fe4ef5f56de5 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -298,8 +298,14 @@ impl PyVersion { .expect("could not create initial worker dir"); let logs = String::new(); - // let v_with_dot = self.to_string_with_dot(); - let mut child_cmd = Command::new(UV_PATH.as_str()); + + #[cfg(windows)] + let uv_cmd = "uv"; + + #[cfg(unix)] + let uv_cmd = UV_PATH.as_str(); + + let mut child_cmd = Command::new(uv_cmd); child_cmd .args(["python", "install", v, "--python-preference=only-managed"]) // TODO: Do we need these? @@ -327,9 +333,13 @@ impl PyVersion { .await } async fn find_python(self) -> error::Result> { - // let mut logs = String::new(); - // let v_with_dot = self.to_string_with_dot(); - let mut child_cmd = Command::new(UV_PATH.as_str()); + #[cfg(windows)] + let uv_cmd = "uv"; + + #[cfg(unix)] + let uv_cmd = UV_PATH.as_str(); + + let mut child_cmd = Command::new(uv_cmd); let output = child_cmd // .current_dir(job_dir) .args([ @@ -1517,7 +1527,6 @@ async fn handle_python_deps( additional_python_paths.append(&mut venv_path); } - // TODO: Annotated version should always be equal to final_version Ok((final_version, additional_python_paths)) }