From d92cc15ee4f9c994cee6c0adea84883f4aa1f23b Mon Sep 17 00:00:00 2001 From: pyranota Date: Sun, 29 Dec 2024 17:27:57 +0300 Subject: [PATCH 1/8] fix(python): Cancel installation and start again within 1s caused module not found --- backend/Cargo.lock | 121 ++++++++++++------ backend/Cargo.toml | 1 + backend/windmill-worker/Cargo.toml | 1 + .../windmill-worker/src/python_executor.rs | 14 +- 4 files changed, 97 insertions(+), 40 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 30ba46c4c192e..2170eaf760cfa 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -39,7 +39,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cipher", "cpufeatures", "opaque-debug", @@ -62,7 +62,7 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "const-random", "getrandom 0.2.15", "once_cell", @@ -986,7 +986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cfg-if", + "cfg-if 1.0.0", "libc", "miniz_oxide 0.8.2", "object", @@ -1210,7 +1210,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if", + "cfg-if 1.0.0", "constant_time_eq", ] @@ -1554,6 +1554,12 @@ dependencies = [ "nom", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -1744,7 +1750,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen", ] @@ -1908,7 +1914,7 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -2042,7 +2048,7 @@ version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", @@ -2139,7 +2145,7 @@ version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "hashbrown 0.14.5", "lock_api", "once_cell", @@ -2902,7 +2908,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dirs-sys-next", ] @@ -3115,7 +3121,7 @@ version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -3166,6 +3172,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + [[package]] name = "error_reporter" version = "1.0.0" @@ -3187,7 +3202,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "home", "windows-sys 0.48.0", ] @@ -3258,7 +3273,7 @@ version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "libredox", "windows-sys 0.59.0", @@ -3744,7 +3759,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -3755,7 +3770,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -3896,7 +3911,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "bytemuck", - "cfg-if", + "cfg-if 1.0.0", "crunchy", "num-traits", "rand 0.8.5", @@ -3999,7 +4014,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" dependencies = [ "async-trait", - "cfg-if", + "cfg-if 1.0.0", "data-encoding", "enum-as-inner", "futures-channel", @@ -4023,7 +4038,7 @@ version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "futures-util", "hickory-proto", "ipconfig", @@ -4585,7 +4600,7 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -4846,7 +4861,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "windows-targets 0.52.6", ] @@ -5160,7 +5175,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "digest 0.10.7", ] @@ -5405,6 +5420,19 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if 0.1.10", + "libc", + "void", +] + [[package]] name = "nix" version = "0.27.1" @@ -5412,7 +5440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ "bitflags 2.6.0", - "cfg-if", + "cfg-if 1.0.0", "libc", ] @@ -5632,6 +5660,18 @@ dependencies = [ "walkdir", ] +[[package]] +name = "ofiles" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6d996a29712d694df329d19f0e18632466a42c9de4aab73dca087a1a4c1bd7d" +dependencies = [ + "error-chain", + "glob", + "log", + "nix 0.17.0", +] + [[package]] name = "once_cell" version = "1.20.2" @@ -5721,7 +5761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ "bitflags 2.6.0", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -5940,7 +5980,7 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall 0.5.8", "smallvec", @@ -6433,7 +6473,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fnv", "lazy_static", "memchr", @@ -7059,7 +7099,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.0", "getrandom 0.2.15", "libc", "spin 0.9.8", @@ -7857,7 +7897,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] @@ -7869,7 +7909,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", "opaque-debug", @@ -7881,7 +7921,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] @@ -8402,7 +8442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "psm", "windows-sys 0.59.0", @@ -8562,7 +8602,7 @@ checksum = "12d0a8eaaf1606c9207077d75828008cb2dfb51b095a766bd2b72ef893576e31" dependencies = [ "ast_node", "better_scoped_tls", - "cfg-if", + "cfg-if 1.0.0", "either", "from_variant", "new_debug_unreachable", @@ -9150,7 +9190,7 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand 2.3.0", "once_cell", "rustix", @@ -9221,7 +9261,7 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", ] @@ -9989,7 +10029,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "rand 0.8.5", "static_assertions", ] @@ -10383,6 +10423,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "vsimd" version = "0.8.0" @@ -10438,7 +10484,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -10463,7 +10509,7 @@ version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -11190,8 +11236,9 @@ dependencies = [ "mappable-rc", "mysql_async", "native-tls", - "nix", + "nix 0.27.1", "object_store", + "ofiles", "once_cell", "opentelemetry", "pem 3.0.4", @@ -11443,7 +11490,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "windows-sys 0.48.0", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 8e86f63caf743..aa3d440e2117d 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -199,6 +199,7 @@ json-pointer = "^0" itertools = "^0" regex = "^1" semver = "^1" +ofiles = "0.2.0" deno_fetch = "0.203.0" deno_tls = "0.166.0" diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 890d3f3a7054b..b29c1691acb78 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -69,6 +69,7 @@ dyn-iter.workspace = true once_cell.workspace = true tokio-postgres.workspace = true bit-vec.workspace = true +ofiles.workspace = true deno_fetch = { workspace = true, optional = true } deno_webidl = { workspace = true, optional = true } deno_web = { workspace = true, optional = true } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 083ff09a63e31..44b40a4c5abf8 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1483,8 +1483,17 @@ pub async fn handle_python_reqs( "{py_prefix}/{}", req.replace(' ', "").replace('/', "").replace(':', "") ); - if metadata(&venv_p).await.is_ok() { - // If dir exists skip installation and push path to output + if metadata(&venv_p).await.is_ok() + // Check if venv_p/.lock is in use + // If it is, it is better to wait for it's release and check if wheel has been installed after + && ofiles::opath(venv_p.clone() + "/.lock") + .unwrap_or_else(|e| { + tracing::error!("Failed to check if any PID has {}/.lock open: {e}", &venv_p); + vec![] + }) + .is_empty() + { + // If dir exists or .lock is held skip installation and push path to output req_paths.push(venv_p); in_cache.push(req.to_string()); } else { @@ -1587,7 +1596,6 @@ pub async fn handle_python_reqs( }); if canceled { - tracing::info!( // If there is listener on other side, workspace_id = %w_id_2, From ca5f85c815616aa3b9d72513b6be2267c9f6e33d Mon Sep 17 00:00:00 2001 From: pyranota Date: Mon, 30 Dec 2024 17:42:30 +0300 Subject: [PATCH 2/8] fix(python): Fix uv install job not exiting on fail Specifically with nmslib installation was hanging without any output, even tho library build was failed. It can be monitored with strace or catp. --- .../windmill-worker/src/python_executor.rs | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 44b40a4c5abf8..34f313e653b87 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1245,7 +1245,6 @@ async fn spawn_uv_install( "--target", venv_p, "--no-cache", - "-q", ] }; @@ -1768,10 +1767,12 @@ pub async fn handle_python_reqs( } }; - let mut stderr = uv_install_proccess + let mut stderr_buf = String::new(); + let mut stderr_pipe = uv_install_proccess .stderr .take() .ok_or(anyhow!("Cannot take stderr from uv_install_proccess"))?; + let stderr_future = stderr_pipe.read_to_string(&mut stderr_buf); if let Some(pid) = pids.lock().await.get_mut(i) { *pid = uv_install_proccess.id(); @@ -1788,9 +1789,12 @@ pub async fn handle_python_reqs( uv_install_proccess.kill().await?; pids.lock().await.get_mut(i).and_then(|e| e.take()); return Err(anyhow::anyhow!("uv pip install was canceled")); - } - // Finished - exitstatus = uv_install_proccess.wait() => match exitstatus { + }, + (_, exitstatus) = async { + // See tokio::process::Child::wait_with_output() for more context + // Sometimes uv_install_proccess.wait() is not exiting if stderr is not awaited before it :/ + (stderr_future.await, uv_install_proccess.wait().await) + } => match exitstatus { Ok(status) => if !status.success() { tracing::warn!( workspace_id = %w_id, @@ -1799,24 +1803,18 @@ pub async fn handle_python_reqs( status.code() ); - let mut buf = String::new(); - stderr.read_to_string(&mut buf).await.unwrap_or_else(|_|{ - buf = "Cannot read stderr to string".to_owned(); - 0 - }); - append_logs( &job_id, w_id, format!( - "\nError while installing {}:\n{buf}", + "\nError while installing {}:\n{stderr_buf}", &req ), db, ) .await; pids.lock().await.get_mut(i).and_then(|e| e.take()); - return Err(anyhow!(buf)); + return Err(anyhow!(stderr_buf)); }, Err(e) => { tracing::error!( From ba66d1839cc411e93bbd5a1a53c15e80db6c2163 Mon Sep 17 00:00:00 2001 From: pyranota Date: Mon, 30 Dec 2024 20:15:40 +0300 Subject: [PATCH 3/8] Replace ofiles approach with mutex --- backend/Cargo.lock | 121 ++++++------------ backend/Cargo.toml | 1 - backend/windmill-worker/Cargo.toml | 1 - .../windmill-worker/src/python_executor.rs | 18 +-- 4 files changed, 44 insertions(+), 97 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 2170eaf760cfa..30ba46c4c192e 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -39,7 +39,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cipher", "cpufeatures", "opaque-debug", @@ -62,7 +62,7 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "const-random", "getrandom 0.2.15", "once_cell", @@ -986,7 +986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cfg-if 1.0.0", + "cfg-if", "libc", "miniz_oxide 0.8.2", "object", @@ -1210,7 +1210,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if 1.0.0", + "cfg-if", "constant_time_eq", ] @@ -1554,12 +1554,6 @@ dependencies = [ "nom", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -1750,7 +1744,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen", ] @@ -1914,7 +1908,7 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2048,7 +2042,7 @@ version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", @@ -2145,7 +2139,7 @@ version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "hashbrown 0.14.5", "lock_api", "once_cell", @@ -2908,7 +2902,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -3121,7 +3115,7 @@ version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -3172,15 +3166,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "error-chain" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -dependencies = [ - "version_check", -] - [[package]] name = "error_reporter" version = "1.0.0" @@ -3202,7 +3187,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "home", "windows-sys 0.48.0", ] @@ -3273,7 +3258,7 @@ version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "libredox", "windows-sys 0.59.0", @@ -3759,7 +3744,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -3770,7 +3755,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -3911,7 +3896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "bytemuck", - "cfg-if 1.0.0", + "cfg-if", "crunchy", "num-traits", "rand 0.8.5", @@ -4014,7 +3999,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" dependencies = [ "async-trait", - "cfg-if 1.0.0", + "cfg-if", "data-encoding", "enum-as-inner", "futures-channel", @@ -4038,7 +4023,7 @@ version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "futures-util", "hickory-proto", "ipconfig", @@ -4600,7 +4585,7 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -4861,7 +4846,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "windows-targets 0.52.6", ] @@ -5175,7 +5160,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "digest 0.10.7", ] @@ -5420,19 +5405,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" -[[package]] -name = "nix" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" -dependencies = [ - "bitflags 1.3.2", - "cc", - "cfg-if 0.1.10", - "libc", - "void", -] - [[package]] name = "nix" version = "0.27.1" @@ -5440,7 +5412,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ "bitflags 2.6.0", - "cfg-if 1.0.0", + "cfg-if", "libc", ] @@ -5660,18 +5632,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "ofiles" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6d996a29712d694df329d19f0e18632466a42c9de4aab73dca087a1a4c1bd7d" -dependencies = [ - "error-chain", - "glob", - "log", - "nix 0.17.0", -] - [[package]] name = "once_cell" version = "1.20.2" @@ -5761,7 +5721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ "bitflags 2.6.0", - "cfg-if 1.0.0", + "cfg-if", "foreign-types", "libc", "once_cell", @@ -5980,7 +5940,7 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall 0.5.8", "smallvec", @@ -6473,7 +6433,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fnv", "lazy_static", "memchr", @@ -7099,7 +7059,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", "getrandom 0.2.15", "libc", "spin 0.9.8", @@ -7897,7 +7857,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.10.7", ] @@ -7909,7 +7869,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.9.0", "opaque-debug", @@ -7921,7 +7881,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.10.7", ] @@ -8442,7 +8402,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "psm", "windows-sys 0.59.0", @@ -8602,7 +8562,7 @@ checksum = "12d0a8eaaf1606c9207077d75828008cb2dfb51b095a766bd2b72ef893576e31" dependencies = [ "ast_node", "better_scoped_tls", - "cfg-if 1.0.0", + "cfg-if", "either", "from_variant", "new_debug_unreachable", @@ -9190,7 +9150,7 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fastrand 2.3.0", "once_cell", "rustix", @@ -9261,7 +9221,7 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", ] @@ -10029,7 +9989,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "rand 0.8.5", "static_assertions", ] @@ -10423,12 +10383,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - [[package]] name = "vsimd" version = "0.8.0" @@ -10484,7 +10438,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -10509,7 +10463,7 @@ version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -11236,9 +11190,8 @@ dependencies = [ "mappable-rc", "mysql_async", "native-tls", - "nix 0.27.1", + "nix", "object_store", - "ofiles", "once_cell", "opentelemetry", "pem 3.0.4", @@ -11490,7 +11443,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "windows-sys 0.48.0", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index aa3d440e2117d..8e86f63caf743 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -199,7 +199,6 @@ json-pointer = "^0" itertools = "^0" regex = "^1" semver = "^1" -ofiles = "0.2.0" deno_fetch = "0.203.0" deno_tls = "0.166.0" diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index b29c1691acb78..890d3f3a7054b 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -69,7 +69,6 @@ dyn-iter.workspace = true once_cell.workspace = true tokio-postgres.workspace = true bit-vec.workspace = true -ofiles.workspace = true deno_fetch = { workspace = true, optional = true } deno_webidl = { workspace = true, optional = true } deno_web = { workspace = true, optional = true } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 34f313e653b87..4e3094db8482f 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -15,7 +15,7 @@ use tokio::{ fs::{metadata, DirBuilder, File}, io::AsyncReadExt, process::Command, - sync::Semaphore, + sync::{RwLock, Semaphore}, task, }; use uuid::Uuid; @@ -35,6 +35,8 @@ use windmill_common::variables::get_secret_value_as_admin; use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { + static ref BUSY_VENV_P: RwLock> = RwLock::new(HashSet::new()); + static ref PYTHON_PATH: String = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); @@ -1482,15 +1484,7 @@ pub async fn handle_python_reqs( "{py_prefix}/{}", req.replace(' ', "").replace('/', "").replace(':', "") ); - if metadata(&venv_p).await.is_ok() - // Check if venv_p/.lock is in use - // If it is, it is better to wait for it's release and check if wheel has been installed after - && ofiles::opath(venv_p.clone() + "/.lock") - .unwrap_or_else(|e| { - tracing::error!("Failed to check if any PID has {}/.lock open: {e}", &venv_p); - vec![] - }) - .is_empty() + if metadata(&venv_p).await.is_ok() && !BUSY_VENV_P.read().await.contains(&venv_p) { // If dir exists or .lock is held skip installation and push path to output req_paths.push(venv_p); @@ -1710,6 +1704,7 @@ pub async fn handle_python_reqs( ); let start = std::time::Instant::now(); + BUSY_VENV_P.write().await.insert(venv_p.clone()); #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if is_not_pro { if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { @@ -1887,8 +1882,9 @@ pub async fn handle_python_reqs( ); } } else { - req_paths.push(venv_p); + req_paths.push(venv_p.clone()); } + BUSY_VENV_P.write().await.remove(&venv_p); } if has_work { From 25c55ea87145158f326e263cd18dc94b057de52f Mon Sep 17 00:00:00 2001 From: pyranota Date: Mon, 30 Dec 2024 20:19:06 +0300 Subject: [PATCH 4/8] Small refactor --- backend/windmill-worker/src/python_executor.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 4e3094db8482f..22ab8cef0cb4a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1484,9 +1484,8 @@ pub async fn handle_python_reqs( "{py_prefix}/{}", req.replace(' ', "").replace('/', "").replace(':', "") ); - if metadata(&venv_p).await.is_ok() && !BUSY_VENV_P.read().await.contains(&venv_p) - { - // If dir exists or .lock is held skip installation and push path to output + if metadata(&venv_p).await.is_ok() && !BUSY_VENV_P.read().await.contains(&venv_p) { + BUSY_VENV_P.write().await.insert(venv_p.clone()); req_paths.push(venv_p); in_cache.push(req.to_string()); } else { @@ -1704,7 +1703,6 @@ pub async fn handle_python_reqs( ); let start = std::time::Instant::now(); - BUSY_VENV_P.write().await.insert(venv_p.clone()); #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if is_not_pro { if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { From 7a55c768457009e04604ece53b2741b0832f52c4 Mon Sep 17 00:00:00 2001 From: pyranota Date: Mon, 30 Dec 2024 20:20:57 +0300 Subject: [PATCH 5/8] Return space for consistency --- backend/windmill-worker/src/python_executor.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 22ab8cef0cb4a..150c614a74d28 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1588,6 +1588,7 @@ pub async fn handle_python_reqs( }); if canceled { + tracing::info!( // If there is listener on other side, workspace_id = %w_id_2, From bf9c25cfd0618a563c260400968749f9c8f15249 Mon Sep 17 00:00:00 2001 From: Pyranota Date: Fri, 3 Jan 2025 16:33:47 +0300 Subject: [PATCH 6/8] Revert incorrect fix --- backend/windmill-worker/src/python_executor.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 150c614a74d28..fc95a5d49a10a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -35,7 +35,6 @@ use windmill_common::variables::get_secret_value_as_admin; use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { - static ref BUSY_VENV_P: RwLock> = RwLock::new(HashSet::new()); static ref PYTHON_PATH: String = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); @@ -1484,8 +1483,7 @@ pub async fn handle_python_reqs( "{py_prefix}/{}", req.replace(' ', "").replace('/', "").replace(':', "") ); - if metadata(&venv_p).await.is_ok() && !BUSY_VENV_P.read().await.contains(&venv_p) { - BUSY_VENV_P.write().await.insert(venv_p.clone()); + if metadata(&venv_p).await.is_ok() { req_paths.push(venv_p); in_cache.push(req.to_string()); } else { @@ -1881,9 +1879,8 @@ pub async fn handle_python_reqs( ); } } else { - req_paths.push(venv_p.clone()); + req_paths.push(venv_p); } - BUSY_VENV_P.write().await.remove(&venv_p); } if has_work { From 57807531c4f8ce0c9a6b881c2dabf91ee62e1be5 Mon Sep 17 00:00:00 2001 From: Pyranota Date: Fri, 3 Jan 2025 16:34:34 +0300 Subject: [PATCH 7/8] Fix module not found --- backend/windmill-worker/src/python_executor.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index fc95a5d49a10a..d8b2015946778 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -7,6 +7,7 @@ use std::{ }; use anyhow::anyhow; +use futures::lock::Mutex; use itertools::Itertools; use regex::Regex; use serde_json::value::RawValue; @@ -35,6 +36,7 @@ use windmill_common::variables::get_secret_value_as_admin; use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { + static ref BUSY_WITH_UV_INSTALL: Mutex<()> = Mutex::new(()); static ref PYTHON_PATH: String = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); @@ -1354,6 +1356,7 @@ pub async fn handle_python_reqs( mut no_uv_install: bool, is_ansible: bool, ) -> error::Result> { + let lock = BUSY_WITH_UV_INSTALL.lock().await; let counter_arc = Arc::new(tokio::sync::Mutex::new(0)); // Append logs with line like this: // [9/21] + requests==2.32.3 << (S3) | in 57ms @@ -1517,6 +1520,12 @@ pub async fn handle_python_reqs( let pids = Arc::new(tokio::sync::Mutex::new(vec![None; total_to_install])); let mem_peak_thread_safe = Arc::new(tokio::sync::Mutex::new(0)); { + // when we cancel the job, it has up to 1 second window before actually getting cancelled + // Thus the directory with wheel in windmill's cache cleaned only after that. + // If we manage to start new job during that period windmill might see that wanted wheel is already there (because we have not cleaned it yet) + // and write it to installed wheels, meanwhile previous job will clean that wheel. + // To fix that we create lock, which will pipeline all uv installs on worker + let _lock = lock; let pids = pids.clone(); let mem_peak_thread_safe = mem_peak_thread_safe.clone(); tokio::spawn(async move { From c912150bce294bae4255c6fed8cf138065bf8625 Mon Sep 17 00:00:00 2001 From: Pyranota Date: Fri, 3 Jan 2025 16:50:15 +0300 Subject: [PATCH 8/8] Remove unused import --- backend/windmill-worker/src/python_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index fc95a5d49a10a..a513dad5dcd3e 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -15,7 +15,7 @@ use tokio::{ fs::{metadata, DirBuilder, File}, io::AsyncReadExt, process::Command, - sync::{RwLock, Semaphore}, + sync::Semaphore, task, }; use uuid::Uuid;