From c23ae142bb63184f5364426365348e9540f1a5eb Mon Sep 17 00:00:00 2001 From: scosman Date: Thu, 9 Jan 2025 21:06:20 -0500 Subject: [PATCH] Big perf improvement on loading. Our OpenAI "is server up" check was retrying (default openai client behaviour), with sleeps between retries! Really slowed things down. Especially since local servers (LiteLLM, etc) are often down if app closed. Now with retry=0 it's speedy --- app/desktop/studio_server/provider_api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/desktop/studio_server/provider_api.py b/app/desktop/studio_server/provider_api.py index 68440c6..1356aeb 100644 --- a/app/desktop/studio_server/provider_api.py +++ b/app/desktop/studio_server/provider_api.py @@ -633,6 +633,10 @@ def openai_compatible_providers_load_cache() -> OpenAICompatibleProviderCache | openai_client = openai.OpenAI( api_key=api_key, base_url=base_url, + # Important: max_retries must be 0 for performance. + # It's common for these servers to be down sometimes (could be local app that isn't running) + # OpenAI client will retry a few times, with a sleep in between! Big loading perf hit. + max_retries=0, ) try: