Skip to content

Commit

Permalink
feat(router.py): add 'get_model_info' helper function to get the mode…
Browse files Browse the repository at this point in the history
…l info for a specific model, based on it's id
  • Loading branch information
krrishdholakia committed May 3, 2024
1 parent fdc4fdb commit 91971fa
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 8 deletions.
19 changes: 12 additions & 7 deletions litellm/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -2590,6 +2590,16 @@ def get_deployment(self, model_id: str):
return model
return None

def get_model_info(self, id: str) -> Optional[dict]:
"""
For a given model id, return the model info
"""
for model in self.model_list:
if "model_info" in model and "id" in model["model_info"]:
if id == model["model_info"]["id"]:
return model
return None

def get_model_ids(self):
ids = []
for model in self.model_list:
Expand Down Expand Up @@ -2904,15 +2914,10 @@ def _common_checks_available_deployment(
m for m in self.model_list if m["litellm_params"]["model"] == model
]

verbose_router_logger.debug(
f"initial list of deployments: {healthy_deployments}"
)
litellm.print_verbose(f"initial list of deployments: {healthy_deployments}")

verbose_router_logger.debug(
f"healthy deployments: length {len(healthy_deployments)} {healthy_deployments}"
)
if len(healthy_deployments) == 0:
raise ValueError(f"No healthy deployment available, passed model={model}")
raise ValueError(f"No healthy deployment available, passed model={model}. ")
if litellm.model_alias_map and model in litellm.model_alias_map:
model = litellm.model_alias_map[
model
Expand Down
4 changes: 3 additions & 1 deletion litellm/router_strategy/lowest_tpm_rpm_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,12 @@ def pre_call_check(self, deployment: Dict) -> Optional[Dict]:
model=deployment.get("litellm_params", {}).get("model"),
response=httpx.Response(
status_code=429,
content="{} rpm limit={}. current usage={}".format(
content="{} rpm limit={}. current usage={}. id={}, model_group={}. Get the model info by calling 'router.get_model_info(id)".format(
RouterErrors.user_defined_ratelimit_error.value,
deployment_rpm,
local_result,
model_id,
deployment.get("model_name", ""),
),
request=httpx.Request(method="tpm_rpm_limits", url="https://github.com/BerriAI/litellm"), # type: ignore
),
Expand Down

0 comments on commit 91971fa

Please sign in to comment.