Merge pull request stanfordnlp#1012 from omkar-sh/bug/vllm_extra_para…

…m_fix bug(dspy): removed extra params from kwargs sent to vllm pydantic model
inayet · May 15, 2024 · 9c1fff9 · 9c1fff9
2 parents 17d76b7 + ec96941
commit 9c1fff9
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py
@@ -154,6 +154,10 @@ def _generate(self, prompt, **kwargs):
             messages = [{"role": "user", "content": prompt}]
             if system_prompt:
                 messages.insert(0, {"role": "system", "content": system_prompt})
+
+            kwargs.pop("port", None)
+            kwargs.pop("url", None)
+
             payload = {
                 "model": self.kwargs["model"],
                 "messages": messages,
@@ -180,6 +184,9 @@ def _generate(self, prompt, **kwargs):
                 print("Failed to parse JSON response:", response.text)
                 raise Exception("Received invalid JSON response from server")
         else:
+            kwargs.pop("port", None)
+            kwargs.pop("url", None)
+
             payload = {
                 "model": self.kwargs["model"],
                 "prompt": prompt,