From 4992ea39a6f131e5efcdb61997f64e254105ee31 Mon Sep 17 00:00:00 2001 From: Lorenzo Pacchiardi Date: Wed, 27 Mar 2024 14:03:22 +0000 Subject: [PATCH] Fix specifying API arguments from the CLI --- evals/cli/oaieval.py | 3 +++ evals/registry.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/evals/cli/oaieval.py b/evals/cli/oaieval.py index 1bf725e2e8..ef8b1c266c 100644 --- a/evals/cli/oaieval.py +++ b/evals/cli/oaieval.py @@ -138,6 +138,9 @@ def run(args: OaiEvalArguments, registry: Optional[Registry] = None) -> str: # If the user provided an argument to --completion_args, parse it into a dict here, to be passed to the completion_fn creation **kwargs completion_args = args.completion_args.split(",") additonal_completion_args = {k: v for k, v in (kv.split("=") for kv in completion_args if kv)} + # if the user provides temperature, transform that into a number: + if "temperature" in additonal_completion_args: + additonal_completion_args["temperature"] = float(additonal_completion_args["temperature"]) completion_fns = args.completion_fn.split(",") completion_fn_instances = [ diff --git a/evals/registry.py b/evals/registry.py index 48a62116f7..765d091988 100644 --- a/evals/registry.py +++ b/evals/registry.py @@ -121,9 +121,9 @@ def make_completion_fn( n_ctx = n_ctx_from_model_name(name) if is_chat_model(name): - return OpenAIChatCompletionFn(model=name, n_ctx=n_ctx, **kwargs) + return OpenAIChatCompletionFn(model=name, n_ctx=n_ctx, extra_options=kwargs) elif name in self.api_model_ids: - return OpenAICompletionFn(model=name, n_ctx=n_ctx, **kwargs) + return OpenAICompletionFn(model=name, n_ctx=n_ctx, extra_options=kwargs) # No match, so try to find a completion-fn-id in the registry spec = self.get_completion_fn(name)