From cc115fab2a827b0e9616070cb1d706f63599cc3d Mon Sep 17 00:00:00 2001 From: juberti Date: Thu, 18 Jul 2024 18:28:33 -0700 Subject: [PATCH] Groq models are fp8 --- llm_benchmark_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_benchmark_suite.py b/llm_benchmark_suite.py index 0072d1d..7ba0b49 100644 --- a/llm_benchmark_suite.py +++ b/llm_benchmark_suite.py @@ -354,7 +354,7 @@ def _text_models(): _FireworksLlm( "accounts/fireworks/models/mixtral-8x7b-instruct-hf", MIXTRAL_8X7B_INSTRUCT ), - _GroqLlm("mixtral-8x7b-32768", MIXTRAL_8X7B_INSTRUCT), + _GroqLlm("mixtral-8x7b-32768", MIXTRAL_8X7B_INSTRUCT_FP8), _NvidiaLlm("mistralai/mixtral-8x7b-instruct-v0.1", MIXTRAL_8X7B_INSTRUCT), _OctoLlm("mixtral-8x7b-instruct", MIXTRAL_8X7B_INSTRUCT), _TogetherLlm("mistralai/Mixtral-8x7B-Instruct-v0.1", MIXTRAL_8X7B_INSTRUCT),