fix(llama.cpp): enable cont batching when parallel is set (mudler#1622)

Signed-off-by: Ettore Di Giacinto <[email protected]>
dave-gray101 · Jan 21, 2024 · 697c769 · 697c769
1 parent 94261b1
commit 697c769
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/backend/cpp/llama/grpc-server.cpp b/backend/cpp/llama/grpc-server.cpp
@@ -2465,10 +2465,10 @@ static void params_parse(const backend::ModelOptions* request,
     const char *env_parallel = std::getenv("LLAMACPP_PARALLEL");
     if (env_parallel != NULL) {
         params.n_parallel = std::stoi(env_parallel);
+        params.cont_batching = true;
     } else {
         params.n_parallel = 1;
     }
-    params.cont_batching = true;
     // TODO: Add yarn
 
     if (!request->tensorsplit().empty()) {