feat: change n_batch default to 2048 (#63)

Co-authored-by: vansangpfiev <[email protected]>
janhq · May 28, 2024 · dc3fe71 · dc3fe71
1 parent d22ee56
commit dc3fe71
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/llama_engine.cc b/src/llama_engine.cc
@@ -327,9 +327,9 @@ bool LlamaEngine::LoadModelImpl(std::shared_ptr<Json::Value> jsonBody) {
     params.n_ctx = jsonBody->get("ctx_len", 2048).asInt();
     params.embedding = jsonBody->get("embedding", true).asBool();
     model_type = jsonBody->get("model_type", "llm").asString();
-    // Check if n_parallel exists in jsonBody, if not, set to drogon_thread
-    params.n_batch = jsonBody->get("n_batch", 512).asInt();
+    params.n_batch = jsonBody->get("n_batch", 2048).asInt();
     params.n_ubatch = jsonBody->get("n_ubatch", params.n_batch).asInt();
+    // Check if n_parallel exists in jsonBody, if not, set to drogon_thread
     params.n_parallel = jsonBody->get("n_parallel", 1).asInt();
     params.n_threads =
         jsonBody->get("cpu_threads", std::thread::hardware_concurrency())