feat(python): Multiple runtime versions (#4579)

* feat: Handle `pip install` by `uv` Dirty and untested, but already something working * Integrate with NSJAIL and prepare fallbacks * Refactor fallback no_uv disable compile and install where no_uv_install and no_uv_compile are a bit more specific * Remove `--disable-pip-version-check` Reason: warning: pip's `--disable-pip-version-check` has no effect * Fix backend compilation error * Pip fallback overwrite UV's cache * Initially refactor cache (No S3) * Support S3 * Remove unused import * Handle flags for NSJAIL * Return deleted flag * Remove verbose mode and enable link-mode=copy * Granural migration of lockfiles Before i realized we dont need it :) * Initial draft (not-working) * Add fallback * Fix bug preventing uv from installing deps '\n' - Love it * Add verbosity indicator * Iterate on feature - Added instance python version - Rework logic * Fix EE build error error[E0599]: no method named `iter` found for tuple `(PyVersion, std::vec::Vec<std::string::String>)` in the current scope * Support S3 * Support NSJAIL * Refactor `get_python` * Make NSJAIL work [Unsafe] config file missed /proc mount causing install phase to fail * Trigger CI * Clean up * Make Actions build it * Trigger CI #2 * Update Dockerfile and clean up * Change fallbacks now there is only no_uv and NOUV * Expose INSTANCE_PYTHON_VERSION through env variable * Change namings * Include py-version to requirements.in Also add comments and make code much cleaner * Use const for python installation dir It was hardcoded before * Pin preinstalled version * Update python_executor.rs * Up to date branch * Create PYCACHE dirs TODO: PY_TAR_DIRS * Fix after merge * Make it safer * Implement USE_SYSTEM_PYTHON * Implement latest_stable option * Load INSTANCE_PYTHON_VERSION on startup * Check for multiple annotations used * Fix Latest Stable button not pressed if selected * Proper error handling for conflict on multiple annotations * Fix merge conflicts * Preinstall 3.11 and Latest Stable * Preinstall latest stable in non-blocking manner * Fix Warning * Gate preinstall logic behind "python" feature * Handle raw_deps properly * Make it work with nsjail * Revert docker-image.yml * Revert Dockerfile * Cleanup + Fixing * Add windows support --------- Co-authored-by: Ruben Fiszel <[email protected]>
windmill-labs · Jan 24, 2025 · e47dd69 · e47dd69
1 parent 6776b2b
commit e47dd69
Show file tree

Hide file tree

Showing 19 changed files with 802 additions and 106 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -95,6 +95,14 @@ ARG WITH_KUBECTL=true
 ARG WITH_HELM=true
 ARG WITH_GIT=true
 
+# To change latest stable version:
+# 1. Change placeholder in instanceSettings.ts
+# 2. Change LATEST_STABLE_PY in dockerfile
+# 3. Change #[default] annotation for PyVersion in backend
+ARG LATEST_STABLE_PY=3.11.10
+ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_runtime
+ENV UV_PYTHON_PREFERENCE=only-managed
+
 RUN pip install --upgrade pip==24.2
 
 RUN apt-get update \
@@ -161,6 +169,10 @@ ENV GO_PATH=/usr/local/go/bin/go
 # Install UV
 RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
 
+# Preinstall python runtimes
+RUN uv python install 3.11.10
+RUN uv python install $LATEST_STABLE_PY
+
 RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - 
 RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \
     && rm -rf /var/lib/apt/lists/*

diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs
@@ -21,7 +21,7 @@ use rustpython_parser::{
     Parse,
 };
 use sqlx::{Pool, Postgres};
-use windmill_common::error;
+use windmill_common::{error, worker::PythonAnnotations};
 
 const DEF_MAIN: &str = "def main(";
 
@@ -171,14 +171,75 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<String>>
     return Ok(nimports);
 }
 
-#[async_recursion]
 pub async fn parse_python_imports(
     code: &str,
     w_id: &str,
     path: &str,
     db: &Pool<Postgres>,
     already_visited: &mut Vec<String>,
+    annotated_pyv_numeric: &mut Option<u32>,
+) -> error::Result<Vec<String>> {
+    parse_python_imports_inner(
+        code,
+        w_id,
+        path,
+        db,
+        already_visited,
+        annotated_pyv_numeric,
+        &mut annotated_pyv_numeric.and_then(|_| Some(path.to_owned())),
+    )
+    .await
+}
+
+#[async_recursion]
+async fn parse_python_imports_inner(
+    code: &str,
+    w_id: &str,
+    path: &str,
+    db: &Pool<Postgres>,
+    already_visited: &mut Vec<String>,
+    annotated_pyv_numeric: &mut Option<u32>,
+    path_where_annotated_pyv: &mut Option<String>,
 ) -> error::Result<Vec<String>> {
+    let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
+
+    // we pass only if there is none or only one annotation
+
+    // Naive:
+    // 1. Check if there are multiple annotated version
+    // 2. If no, take one and compare with annotated version
+    // 3. We continue if same or replace none with new one
+
+    // Optimized:
+    // 1. Iterate over all annotations compare each with annotated_pyv and replace on flight
+    // 2. If annotated_pyv is different version, throw and error
+
+    // This way we make sure there is no multiple annotations for same script
+    // and we get detailed span on conflicting versions
+
+    let mut check = |is_py_xyz, numeric| -> error::Result<()> {
+        if is_py_xyz {
+            if let Some(v) = annotated_pyv_numeric {
+                if *v != numeric {
+                    return Err(error::Error::from(anyhow::anyhow!(
+                        "Annotated 2 or more different python versions: \n - py{v} at {}\n - py{numeric} at {path}\nIt is possible to use only one.",
+                        path_where_annotated_pyv.clone().unwrap_or("Unknown".to_owned())
+                    )));
+                }
+            } else {
+                *annotated_pyv_numeric = Some(numeric);
+            }
+
+            *path_where_annotated_pyv = Some(path.to_owned());
+        }
+        Ok(())
+    };
+
+    check(py310, 310)?;
+    check(py311, 311)?;
+    check(py312, 312)?;
+    check(py313, 313)?;
+
     let find_requirements = code
         .lines()
         .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:"));
@@ -225,11 +286,21 @@ pub async fn parse_python_imports(
                 .fetch_optional(db)
                 .await?
                 .unwrap_or_else(|| "".to_string());
+
                 if already_visited.contains(&rpath) {
                     vec![]
                 } else {
                     already_visited.push(rpath.clone());
-                    parse_python_imports(&code, w_id, &rpath, db, already_visited).await?
+                    parse_python_imports_inner(
+                        &code,
+                        w_id,
+                        &rpath,
+                        db,
+                        already_visited,
+                        annotated_pyv_numeric,
+                        path_where_annotated_pyv,
+                    )
+                    .await?
                 }
             } else {
                 vec![replace_import(n.to_string())]

diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs
@@ -25,6 +25,7 @@ def main():
             "f/foo/bar",
             &db,
             &mut already_visited,
+            &mut None,
         )
         .await?;
         // println!("{}", serde_json::to_string(&r)?);
@@ -57,6 +58,7 @@ def main():
             "f/foo/bar",
             &db,
             &mut already_visited,
+            &mut None,
         )
         .await?;
         println!("{}", serde_json::to_string(&r)?);
@@ -87,6 +89,7 @@ def main():
             "f/foo/bar",
             &db,
             &mut already_visited,
+            &mut None,
         )
         .await?;
         println!("{}", serde_json::to_string(&r)?);

diff --git a/backend/src/main.rs b/backend/src/main.rs
@@ -9,8 +9,9 @@
 use anyhow::Context;
 use monitor::{
     load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
-    reload_nuget_config_setting, reload_timeout_wait_result_setting,
-    send_current_log_file_to_object_store, send_logs_to_object_store,
+    reload_instance_python_version_setting, reload_nuget_config_setting,
+    reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
+    send_logs_to_object_store,
 };
 use rand::Rng;
 use sqlx::{postgres::PgListener, Pool, Postgres};
@@ -35,7 +36,7 @@ use windmill_common::{
         CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
         DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
         EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
-        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
+        INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
         LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
         NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING,
         REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
@@ -69,8 +70,9 @@ use windmill_worker::{
     get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
     BUN_DEPSTAR_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS,
     DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR,
-    POWERSHELL_CACHE_DIR, PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR,
-    TMP_LOGS_DIR, UV_CACHE_DIR,
+    POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR,
+    RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR,
+    TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
 };
 
 use crate::monitor::{
@@ -765,6 +767,9 @@ Windmill Community Edition {GIT_VERSION}
                                                 PIP_INDEX_URL_SETTING => {
                                                     reload_pip_index_url_setting(&db).await
                                                 },
+                                                INSTANCE_PYTHON_VERSION_SETTING => {
+                                                    reload_instance_python_version_setting(&db).await
+                                                },
                                                 NPM_CONFIG_REGISTRY_SETTING => {
                                                     reload_npm_config_registry_setting(&db).await
                                                 },
@@ -1016,12 +1021,18 @@ pub async fn run_workers(
         TMP_LOGS_DIR,
         UV_CACHE_DIR,
         TAR_PIP_CACHE_DIR,
-        TAR_PY311_CACHE_DIR,
         DENO_CACHE_DIR,
         DENO_CACHE_DIR_DEPS,
         DENO_CACHE_DIR_NPM,
         BUN_CACHE_DIR,
+        PY310_CACHE_DIR,
         PY311_CACHE_DIR,
+        PY312_CACHE_DIR,
+        PY313_CACHE_DIR,
+        TAR_PY310_CACHE_DIR,
+        TAR_PY311_CACHE_DIR,
+        TAR_PY312_CACHE_DIR,
+        TAR_PY313_CACHE_DIR,
         PIP_CACHE_DIR,
         BUN_DEPSTAR_CACHE_DIR,
         BUN_BUNDLE_CACHE_DIR,

diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs
@@ -41,10 +41,10 @@ use windmill_common::{
         BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
         CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
         DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
-        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
-        JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
-        MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING,
-        OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
+        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
+        LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
+        NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
@@ -68,8 +68,8 @@ use windmill_common::{
 use windmill_queue::cancel_job;
 use windmill_worker::{
     create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender,
-    SendResult, BUNFIG_INSTALL_SCOPES, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY,
-    NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
+    SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR,
+    NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
 };
 
 #[cfg(feature = "parquet")]
@@ -198,6 +198,7 @@ pub async fn initial_load(
         reload_pip_index_url_setting(&db).await;
         reload_npm_config_registry_setting(&db).await;
         reload_bunfig_install_scopes_setting(&db).await;
+        reload_instance_python_version_setting(&db).await;
         reload_nuget_config_setting(&db).await;
     }
 }
@@ -908,6 +909,16 @@ pub async fn reload_pip_index_url_setting(db: &DB) {
     .await;
 }
 
+pub async fn reload_instance_python_version_setting(db: &DB) {
+    reload_option_setting_with_tracing(
+        db,
+        INSTANCE_PYTHON_VERSION_SETTING,
+        "INSTANCE_PYTHON_VERSION",
+        INSTANCE_PYTHON_VERSION.clone(),
+    )
+    .await;
+}
+
 pub async fn reload_npm_config_registry_setting(db: &DB) {
     reload_option_setting_with_tracing(
         db,

diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs
@@ -66,6 +66,8 @@ pub enum Error {
     AiError(String),
     #[error("{0}")]
     AlreadyCompleted(String),
+    #[error("Find python error: {0}")]
+    FindPythonError(String),
     #[error("{0}")]
     Utf8(#[from] std::string::FromUtf8Error),
     #[error("Encoding/decoding error: {0}")]

diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs
@@ -14,6 +14,7 @@ pub const NUGET_CONFIG_SETTING: &str = "nuget_config";
 
 pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url";
 pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url";
+pub const INSTANCE_PYTHON_VERSION_SETTING: &str = "instance_python_version";
 pub const SCIM_TOKEN_SETTING: &str = "scim_token";
 pub const SAML_METADATA_SETTING: &str = "saml_metadata";
 pub const SMTP_SETTING: &str = "smtp_settings";
@@ -39,7 +40,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret";
 pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";
 pub const OTEL_SETTING: &str = "otel";
 
-pub const ENV_SETTINGS: [&str; 54] = [
+pub const ENV_SETTINGS: [&str; 55] = [
     "DISABLE_NSJAIL",
     "MODE",
     "NUM_WORKERS",
@@ -62,6 +63,7 @@ pub const ENV_SETTINGS: [&str; 54] = [
     "GOPRIVATE",
     "GOPROXY",
     "NETRC",
+    "INSTANCE_PYTHON_VERSION",
     "PIP_INDEX_URL",
     "PIP_EXTRA_INDEX_URL",
     "PIP_TRUSTED_HOST",

diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs
@@ -333,14 +333,18 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
         .flatten()
 }
 
+#[derive(Copy, Clone)]
 #[annotations("#")]
 pub struct PythonAnnotations {
     pub no_cache: bool,
     pub no_uv: bool,
     pub no_uv_install: bool,
     pub no_uv_compile: bool,
-
     pub no_postinstall: bool,
+    pub py310: bool,
+    pub py311: bool,
+    pub py312: bool,
+    pub py313: bool,
 }
 
 #[annotations("//")]

diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto
@@ -20,6 +20,7 @@ clone_newuser: {CLONE_NEWUSER}
 
 keep_caps: true
 keep_env: true
+mount_proc: true
 
 mount {
     src: "/bin"
@@ -79,6 +80,11 @@ mount {
     is_bind: true
     rw: true
 }
+mount {
+    src: "{PY_INSTALL_DIR}"
+    dst: "{PY_INSTALL_DIR}"
+    is_bind: true
+}
 
 mount {
     src: "/dev/urandom"

diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh
@@ -27,6 +27,7 @@ CMD="/usr/local/bin/uv pip install
 --no-color 
 --no-deps 
 --link-mode=copy
+$PY_PATH 
 $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG
 --index-strategy unsafe-best-match
 --system

diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto
@@ -16,6 +16,7 @@ clone_newuser: {CLONE_NEWUSER}
 
 keep_caps: false
 keep_env: true
+mount_proc: true
 
 mount {
     src: "/bin"
@@ -110,6 +111,12 @@ mount {
     is_bind: true
 }
 
+mount {
+    src: "{PY_INSTALL_DIR}"
+    dst: "{PY_INSTALL_DIR}"
+    is_bind: true
+}
+
 mount {
     src: "/dev/urandom"
     dst: "/dev/urandom"