From 75d7397cec176b2cb90dcfd8226dc7e23e0d12dc Mon Sep 17 00:00:00 2001
From: Tobias Schoofs <tobias.schoofs@gmx.net>
Date: Wed, 31 Jan 2024 16:34:52 +0000
Subject: [PATCH 1/2] [fix/issue1] if we don't have a runtime, create one

---
 crates/edgen_rt_llama_cpp/src/lib.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/crates/edgen_rt_llama_cpp/src/lib.rs b/crates/edgen_rt_llama_cpp/src/lib.rs
index 141f2c2..1a5bf94 100644
--- a/crates/edgen_rt_llama_cpp/src/lib.rs
+++ b/crates/edgen_rt_llama_cpp/src/lib.rs
@@ -26,6 +26,7 @@ use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
 use tokio::task::JoinHandle;
 use tokio::time::{interval, MissedTickBehavior};
 use tokio::{select, spawn};
+use tokio::runtime::{Handle, Runtime};
 use tracing::{error, info};
 
 use edgen_core::llm::{
@@ -123,6 +124,14 @@ impl Default for LlamaCppEndpoint {
     fn default() -> Self {
         let models: Arc<DashMap<String, UnloadingModel>> = Default::default();
 
+        let (handle, _rt) = match Handle::try_current() {
+            Ok(h) => (h, None),
+            Err(_) => {
+                let rt = Runtime::new().unwrap();
+                (rt.handle().clone(), Some(rt))
+            },
+        };
+        let _guard = handle.enter();
         let models_clone = models.clone();
         let cleanup_thread = spawn(async move {
             let mut interval = interval(cleanup_interval());

From a5097a99bb8cd487ced48c5e542b902e62f38cb6 Mon Sep 17 00:00:00 2001
From: Tobias Schoofs <tobias.schoofs@gmx.net>
Date: Wed, 31 Jan 2024 18:06:05 +0000
Subject: [PATCH 2/2] [fix/issue1] improved solution: create runtime in
 callback

---
 crates/edgen_rt_llama_cpp/src/lib.rs | 10 ----------
 crates/edgen_server/src/lib.rs       |  2 ++
 2 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/crates/edgen_rt_llama_cpp/src/lib.rs b/crates/edgen_rt_llama_cpp/src/lib.rs
index 1a5bf94..8aa28c7 100644
--- a/crates/edgen_rt_llama_cpp/src/lib.rs
+++ b/crates/edgen_rt_llama_cpp/src/lib.rs
@@ -26,7 +26,6 @@ use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
 use tokio::task::JoinHandle;
 use tokio::time::{interval, MissedTickBehavior};
 use tokio::{select, spawn};
-use tokio::runtime::{Handle, Runtime};
 use tracing::{error, info};
 
 use edgen_core::llm::{
@@ -123,15 +122,6 @@ impl LLMEndpoint for LlamaCppEndpoint {
 impl Default for LlamaCppEndpoint {
     fn default() -> Self {
         let models: Arc<DashMap<String, UnloadingModel>> = Default::default();
-
-        let (handle, _rt) = match Handle::try_current() {
-            Ok(h) => (h, None),
-            Err(_) => {
-                let rt = Runtime::new().unwrap();
-                (rt.handle().clone(), Some(rt))
-            },
-        };
-        let _guard = handle.enter();
         let models_clone = models.clone();
         let cleanup_thread = spawn(async move {
             let mut interval = interval(cleanup_interval());
diff --git a/crates/edgen_server/src/lib.rs b/crates/edgen_server/src/lib.rs
index 4d18661..870d167 100644
--- a/crates/edgen_server/src/lib.rs
+++ b/crates/edgen_server/src/lib.rs
@@ -258,6 +258,8 @@ async fn run_server(args: &cli::Serve) -> bool {
     let flag_clone = reset_flag.clone();
 
     let _callback_handle = SETTINGS.read().await.add_change_callback(move || {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let _guard = rt.enter();
         flag_clone.store(true, Ordering::SeqCst);
         reset_channels.clear();
         block_on(crate::llm::reset_environment());