From 0c19171f579cdd5d2753bd58dcc87b326cb6c09e Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Sat, 4 Jan 2025 01:47:57 +0100 Subject: [PATCH] feat: allow s3 file download/preview from inside apps (#5004) * feat: allow s3 file download/preview from inside apps * improve security + handle image preview * fix build * fix build --- ...5606940a089f6179e5443090afa9aba7c5b24.json | 24 ++++ backend/ee-repo-ref.txt | 2 +- backend/windmill-api/src/apps.rs | 134 +++++++++++++++++- backend/windmill-api/src/job_helpers_ee.rs | 42 ++++++ .../src/lib/components/DisplayResult.svelte | 9 +- .../display/AppDisplayComponent.svelte | 3 +- .../common/fileDownload/FileDownload.svelte | 7 +- 7 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 backend/.sqlx/query-df0454f75e819d7d3d03fef0a7d5606940a089f6179e5443090afa9aba7c5b24.json diff --git a/backend/.sqlx/query-df0454f75e819d7d3d03fef0a7d5606940a089f6179e5443090afa9aba7c5b24.json b/backend/.sqlx/query-df0454f75e819d7d3d03fef0a7d5606940a089f6179e5443090afa9aba7c5b24.json new file mode 100644 index 0000000000000..166dc4a5d85ce --- /dev/null +++ b/backend/.sqlx/query-df0454f75e819d7d3d03fef0a7d5606940a089f6179e5443090afa9aba7c5b24.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM completed_job \n WHERE workspace_id = $2 \n AND (job_kind = 'appscript' OR job_kind = 'preview')\n AND created_by = 'anonymous' \n AND started_at > now() - interval '3 hours'\n AND script_path LIKE $3 || '/%' \n AND result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb \n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "df0454f75e819d7d3d03fef0a7d5606940a089f6179e5443090afa9aba7c5b24" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 474547c202bba..d1a043fc40574 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -586b02014d57f862a5c4313dd1e529d50c315c30 \ No newline at end of file +a1094bec38924de76936392b1148829b02628174 \ No newline at end of file diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 2c4cf5ce07911..7218f98ec0d88 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -20,11 +20,14 @@ use crate::{ #[cfg(feature = "parquet")] use crate::{ job_helpers_ee::{ - get_random_file_name, get_s3_resource, get_workspace_s3_resource, upload_file_from_req, - UploadFileResponse, + download_s3_file_internal, get_random_file_name, get_s3_resource, + get_workspace_s3_resource, load_image_preview_internal, upload_file_from_req, + DownloadFileQuery, LoadImagePreviewQuery, UploadFileResponse, }, users::fetch_api_authed_from_permissioned_as, }; +#[cfg(feature = "parquet")] +use axum::response::Response; use axum::{ extract::{Extension, Json, Path, Query}, response::IntoResponse, @@ -93,6 +96,11 @@ pub fn unauthed_service() -> Router { Router::new() .route("/execute_component/*path", post(execute_component)) .route("/upload_s3_file/*path", post(upload_s3_file_from_app)) + .route("/download_s3_file/*path", get(download_s3_file_from_app)) + .route( + "/load_image_preview/*path", + get(load_s3_file_image_preview_from_app), + ) .route("/public_app/:secret", get(get_public_app_by_secret)) .route("/public_resource/*path", get(get_public_resource)) } @@ -1718,6 +1726,128 @@ async fn upload_s3_file_from_app( return Ok(Json(UploadFileResponse { file_key })); } +#[cfg(not(feature = "parquet"))] +async fn download_s3_file_from_app() -> Result<()> { + return Err(Error::BadRequest( + "This endpoint requires the parquet feature to be enabled".to_string(), + )); +} + +#[cfg(feature = "parquet")] +async fn get_on_behalf_authed_from_app( + db: &DB, + path: &str, + w_id: &str, + opt_authed: &Option, +) -> Result { + let policy_o = sqlx::query_scalar!( + "SELECT policy from app WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(db) + .await?; + + let policy = policy_o + .map(|p| serde_json::from_value::(p).map_err(to_anyhow)) + .transpose()? + .unwrap_or_else(|| Policy { + execution_mode: ExecutionMode::Viewer, + triggerables: None, + triggerables_v2: None, + on_behalf_of: None, + on_behalf_of_email: None, + s3_inputs: None, + }); + + let (username, permissioned_as, email) = + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + + let on_behalf_authed = + fetch_api_authed_from_permissioned_as(permissioned_as, email, &w_id, &db, Some(username)) + .await?; + + Ok(on_behalf_authed) +} + +#[cfg(feature = "parquet")] +async fn check_if_allowed_to_access_s3_file_from_app( + db: &DB, + opt_authed: &Option, + file_key: &str, + w_id: &str, + path: &str, +) -> Result<()> { + // if anonymous, check that the file was the result of an app script ran by an anonymous user in the last 3 hours + // otherwise, if logged in, allow any file (TODO: change that when we implement better s3 policy) + + let allowed = opt_authed.is_some() + || sqlx::query_scalar!( + r#"SELECT EXISTS ( + SELECT 1 FROM completed_job + WHERE workspace_id = $2 + AND (job_kind = 'appscript' OR job_kind = 'preview') + AND created_by = 'anonymous' + AND started_at > now() - interval '3 hours' + AND script_path LIKE $3 || '/%' + AND result @> ('{"s3":"' || $1 || '"}')::jsonb + )"#, + file_key, + w_id, + path, + ) + .fetch_one(db) + .await? + .unwrap_or(false); + + if !allowed { + Err(Error::BadRequest("File restricted".to_string())) + } else { + Ok(()) + } +} + +#[cfg(feature = "parquet")] +async fn download_s3_file_from_app( + OptAuthed(opt_authed): OptAuthed, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> Result { + let path = path.to_path(); + + let on_behalf_authed = get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed).await?; + + check_if_allowed_to_access_s3_file_from_app(&db, &opt_authed, &query.file_key, &w_id, &path) + .await?; + + download_s3_file_internal(on_behalf_authed, &db, None, "", &w_id, query).await +} + +#[cfg(not(feature = "parquet"))] +async fn load_s3_file_image_preview_from_app() -> Result<()> { + return Err(Error::BadRequest( + "This endpoint requires the parquet feature to be enabled".to_string(), + )); +} + +#[cfg(feature = "parquet")] +async fn load_s3_file_image_preview_from_app( + OptAuthed(opt_authed): OptAuthed, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> Result { + let path = path.to_path(); + + let on_behalf_authed = get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed).await?; + + check_if_allowed_to_access_s3_file_from_app(&db, &opt_authed, &query.file_key, &w_id, &path) + .await?; + + load_image_preview_internal(on_behalf_authed, &db, "", &w_id, query).await +} + fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { let permissioned_as = policy .on_behalf_of diff --git a/backend/windmill-api/src/job_helpers_ee.rs b/backend/windmill-api/src/job_helpers_ee.rs index be2dd49ab83ff..7b074a300a2c2 100644 --- a/backend/windmill-api/src/job_helpers_ee.rs +++ b/backend/windmill-api/src/job_helpers_ee.rs @@ -18,11 +18,26 @@ use bytes::Bytes; #[cfg(feature = "parquet")] use futures::Stream; +#[cfg(feature = "parquet")] +use axum::response::Response; +#[cfg(feature = "parquet")] +use serde::Deserialize; + #[derive(Serialize)] pub struct UploadFileResponse { pub file_key: String, } +#[derive(Deserialize)] +pub struct LoadImagePreviewQuery { + pub file_key: String, +} + +#[derive(Deserialize)] +pub struct DownloadFileQuery { + pub file_key: String, +} + pub fn workspaced_service() -> Router { Router::new() } @@ -82,3 +97,30 @@ pub async fn upload_file_internal( "Not implemented in Windmill's Open Source repository".to_string(), )) } + +#[cfg(feature = "parquet")] +pub async fn download_s3_file_internal( + _authed: ApiAuthed, + _db: &DB, + _user_db: Option, + _token: &str, + _w_id: &str, + _query: DownloadFileQuery, +) -> error::Result { + Err(error::Error::InternalErr( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +#[cfg(feature = "parquet")] +pub async fn load_image_preview_internal( + _authed: ApiAuthed, + _db: &DB, + _token: &str, + _w_id: &str, + _query: LoadImagePreviewQuery, +) -> error::Result { + Err(error::Error::InternalErr( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 5e9c4fef8c5ce..b5088c0a5f636 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -47,6 +47,7 @@ export let drawerOpen = false export let nodeId: string | undefined = undefined export let language: string | undefined = undefined + export let appPath: string | undefined = undefined const IMG_MAX_SIZE = 10000000 const TABLE_MAX_SIZE = 5000000 @@ -645,7 +646,7 @@ > {:else if !result?.disable_download} - +