From bb5018fb5b9f61ae00a8c60c5acdd0ab865f6915 Mon Sep 17 00:00:00 2001
From: Yann Dubois <yanndubois96@gmail.com>
Date: Mon, 31 Jul 2023 04:07:41 +0200
Subject: [PATCH] add guanaco-33b-api (#106)

---
 src/alpaca_eval/decoders/huggingface_api.py              | 4 ++--
 src/alpaca_eval/main.py                                  | 5 +++++
 .../models_configs/guanaco-33b-api/configs.yaml          | 9 +++++++++
 3 files changed, 16 insertions(+), 2 deletions(-)
 create mode 100644 src/alpaca_eval/models_configs/guanaco-33b-api/configs.yaml

diff --git a/src/alpaca_eval/decoders/huggingface_api.py b/src/alpaca_eval/decoders/huggingface_api.py
index de985f4f..afff899d 100644
--- a/src/alpaca_eval/decoders/huggingface_api.py
+++ b/src/alpaca_eval/decoders/huggingface_api.py
@@ -18,7 +18,7 @@ def huggingface_api_completions(
     model_name: str,
     gpu: bool = False,
     do_sample: bool = False,
-    num_procs: int = 8,
+    num_procs: int = 1,
     **kwargs,
 ) -> dict[str, list]:
     """Decode with the API from hugging face hub.
@@ -96,7 +96,7 @@ def inference_helper(prompt: str, inference, params, n_retries=100, waiting_time
             elif "Input validation error" in error and "max_new_tokens" in error:
                 params["max_new_tokens"] = int(params["max_new_tokens"] * 0.8)
                 logging.warning(
-                    f"`max_new_tokens` too large. Reducing target length to {params['max_tokens']}, " f"Retrying..."
+                    f"`max_new_tokens` too large. Reducing target length to {params['max_new_tokens']}, " f"Retrying..."
                 )
                 if params["max_new_tokens"] == 0:
                     raise ValueError(f"Error in inference. Full error: {error}")
diff --git a/src/alpaca_eval/main.py b/src/alpaca_eval/main.py
index d94c39cc..b7119f68 100644
--- a/src/alpaca_eval/main.py
+++ b/src/alpaca_eval/main.py
@@ -242,6 +242,11 @@ def evaluate_from_model(
 
     if chunksize is not None and not is_load_outputs:
         logging.info("`is_load_outputs` has to be true to use chunksize. Setting it to True.")
+        is_load_outputs = True
+
+    if chunksize is not None and max_instances is not None:
+        logging.info("cannot use `chunksize` with max_instances. Setting `chunksize` to None.")
+        chunksize = None
 
     model_configs = utils.load_configs(model_configs, relative_to=constants.MODELS_CONFIG_DIR)
     if reference_model_configs is not None:
diff --git a/src/alpaca_eval/models_configs/guanaco-33b-api/configs.yaml b/src/alpaca_eval/models_configs/guanaco-33b-api/configs.yaml
new file mode 100644
index 00000000..1356dfc5
--- /dev/null
+++ b/src/alpaca_eval/models_configs/guanaco-33b-api/configs.yaml
@@ -0,0 +1,9 @@
+guanaco-33b-api:
+  prompt_template: "guanaco-7b/prompt.txt"
+  fn_completions: "huggingface_api_completions"
+  completions_kwargs:
+    model_name: "timdettmers/guanaco-33b-merged"
+    max_new_tokens: 256
+    temperature: 0.7
+    do_sample: True
+    gpu: True
\ No newline at end of file