From aa187f08ae4a85ac8d1a557534d0cb0d7870a91b Mon Sep 17 00:00:00 2001 From: Noah Yoshida Date: Mon, 26 Feb 2024 14:25:47 -0800 Subject: [PATCH 1/2] add in response headers to streaming endpoint --- router/src/server.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/router/src/server.rs b/router/src/server.rs index 08dd4d0d8..ff7da370b 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -554,6 +554,26 @@ async fn generate_stream_with_callback( }); } + let (adapter_source, adapter_parameters) = extract_adapter_params( + req.0.parameters.adapter_id.clone(), + req.0.parameters.adapter_source.clone(), + req.0.parameters.adapter_parameters.clone(), + ); + + let adapter_id_string = adapter_parameters + .adapter_ids + .iter() + .map(|id| id.as_str()) + // filter out base model adapter id + .filter(|id| *id != BASE_MODEL_ADAPTER_ID) + .collect::>() + .join(","); + + if adapter_id_string.len() > 0 { + headers.insert("x-adapter-ids", adapter_id_string.parse().unwrap()); + headers.insert("x-adapter-source", adapter_source.unwrap().parse().unwrap()); + } + let stream = async_stream::stream! { // Inference let mut end_reached = false; From c1640e25540873324995b0041f2bc1bfa62b06be Mon Sep 17 00:00:00 2001 From: Noah Yoshida Date: Mon, 26 Feb 2024 14:30:20 -0800 Subject: [PATCH 2/2] add model id --- router/src/server.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/router/src/server.rs b/router/src/server.rs index ff7da370b..bbd51b935 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -574,6 +574,8 @@ async fn generate_stream_with_callback( headers.insert("x-adapter-source", adapter_source.unwrap().parse().unwrap()); } + headers.insert("x-model-id", MODEL_ID.get().unwrap().parse().unwrap()); + let stream = async_stream::stream! { // Inference let mut end_reached = false;