Add Zephyr 7B evals

tatsu-lab · Oct 26, 2023 · b838d3f · b838d3f
1 parent afdef27
commit b838d3f
Show file tree

Hide file tree

Showing 8 changed files with 29,023 additions and 1 deletion.
diff --git a/results/zephyr-7b-alpha/annotation_alpaca_eval_gpt4.json b/results/zephyr-7b-alpha/annotation_alpaca_eval_gpt4.json
diff --git a/results/zephyr-7b-alpha/model_outputs.json b/results/zephyr-7b-alpha/model_outputs.json
diff --git a/results/zephyr-7b-beta/annotation_alpaca_eval_gpt4.json b/results/zephyr-7b-beta/annotation_alpaca_eval_gpt4.json
diff --git a/results/zephyr-7b-beta/model_outputs.json b/results/zephyr-7b-beta/model_outputs.json
diff --git a/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv b/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv
@@ -6,18 +6,20 @@ ultralm-13b-v2.0-best-of-16,92.29813665,0.940299807,743,62,0,805,community,1720
 xwinlm-13b-v0.1,91.76029963,0.968139439,734,65,2,801,community,1894
 ultralm-13b-best-of-16,91.54228856,0.981927769,736,68,0,804,community,1980
 claude-2,91.35572139,0.989732378,734,69,1,804,minimal,1069
+zephyr-7b-beta,90.59775840597759,1.0287080531312012,727,75,1,803,community,1444
 openchat-v3.1-13b,89.49004975,1.076875475,718,83,3,804,community,1484
 chatgpt,89.36567164,1.078948702,716,83,5,804,minimal,827
 wizardlm-13b-v1.2,89.16562889,1.090425466,714,85,4,803,community,1635
 vicuna-33b-v1.3,88.99253731,1.095692216,713,86,5,804,verified,1479
 claude,88.38509317,1.11448754,707,89,9,805,minimal,1082
-causallm-14b,88.26086956521739,1.1163330437039893,705,89,11,805,community,1391
+causallm-14b,88.26086956521739,1.1163330437039891,705,89,11,805,community,1391
 humpback-llama2-70b,87.93532338,1.154547675,706,96,2,804,community,1822
 xwinlm-7b-v0.1,87.82771536,1.154308696,703,97,1,801,community,1894
 openbuddy-llama2-70b-v10.1,87.67123288,1.150841752,701,96,6,803,community,1077
 openchat-v2-w-13b,87.12686567,1.176919744,699,102,3,804,community,1566
 openbuddy-llama-65b-v8,86.53366584,1.20291824,693,107,2,802,community,1162
 wizardlm-13b-v1.1,86.31840796,1.206321783,692,108,4,804,community,1525
+zephyr-7b-alpha,85.7587064676617,1.2285451680042003,688,113,3,804,community,1302
 cohere,85.05603985,1.255832984,682,119,2,803,community,1715
 openchat-v2-13b,84.9689441,1.257297984,683,120,2,805,community,1564
 humpback-llama-65b,83.70646766,1.307103474,672,130,2,804,community,1269

diff --git a/src/alpaca_eval/models_configs/zephyr-7b-alpha/configs.yaml b/src/alpaca_eval/models_configs/zephyr-7b-alpha/configs.yaml
@@ -0,0 +1,13 @@
+zephyr-7b-alpha:
+  prompt_template: "zephyr-7b-alpha/prompt.txt"
+  fn_completions: "huggingface_local_completions"
+  completions_kwargs:
+    model_name: "HuggingFaceH4/zephyr-7b-alpha"
+    model_kwargs:
+      torch_dtype: 'bfloat16'
+    max_new_tokens: 2048
+    temperature: 0.7
+    top_p: 1.0
+    do_sample: True
+  pretty_name: "Zephyr 7B Alpha"
+  link: "https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha"
diff --git a/src/alpaca_eval/models_configs/zephyr-7b-alpha/prompt.txt b/src/alpaca_eval/models_configs/zephyr-7b-alpha/prompt.txt
@@ -0,0 +1,5 @@
+<|system|>
+</s>
+<|user|>
+{instruction}</s>
+<|assistant|>
diff --git a/src/alpaca_eval/models_configs/zephyr-7b-beta/configs.yaml b/src/alpaca_eval/models_configs/zephyr-7b-beta/configs.yaml
@@ -0,0 +1,14 @@
+zephyr-7b-beta:
+  prompt_template: "zephyr-7b-alpha/prompt.txt"
+  fn_completions: "huggingface_local_completions"
+  completions_kwargs:
+    model_name: "HuggingFaceH4/zephyr-7b-beta"
+    model_kwargs:
+      torch_dtype: 'bfloat16'
+      revision: v8.2
+    max_new_tokens: 2048
+    temperature: 0.7
+    top_p: 1.0
+    do_sample: True
+  pretty_name: "Zephyr 7B Beta"
+  link: "https://huggingface.co/HuggingFaceH4/zephyr-7b-beta"