
    ^h5t                    X   S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKJr  S SKJrJrJrJrJrJrJrJrJrJr  S SKrS SKrS SKJr  S SKJr  S SKJr  S SKrS SK rS SK!rS SKJ"r"  S SK#J$r$  S S	K%J&r&J'r'J(r(  S S
K)J*r*  S SK+J,r,  S SK-J.r/  S SK0J1r1  S SK2J3r3  S SK4J5r5  S SK6J7r7  S SK8J9r9  S SK:J;r;  S SK<J=r=  S SK>J?r?  S SK@JArAJBrB  S SKCJDrD  S SKEJFrF  S SKGJHrHJIrIJJrJJKrKJLrL  S SKMJNrNJOrOJPrP  S SKQJRrS  S SKTJUrUJVrV  S SKWJXrX  S SKYJZrZJ[r[  S SK\J]r]J^r^  S SK_J`r`JaraJbrbJcrc  S S KdJereJfrfJgrgJhrhJiriJjrjJkrkJlrlJmrmJnrnJoroJprpJqrqJrrrJsrsJtrtJuruJvrvJwrw  S S!KxJyry  S S"KzJ{r|  S S#KzJ}r}  S S$K~JrJrJrJrJrJrJr  S%S&KJr  \(       a	  S S'KJr  \rO\r " S( S)\GR                  5      r " S* S+5      rg),    Ndefaultdict)
TYPE_CHECKINGAnyCallableDictListLiteralOptionalTupleUnioncast)AsyncOpenAI)	BaseModel)overload)get_secret_str)verbose_router_logger)	DualCacheInMemoryCache
RedisCache)CustomLogger)!_get_parent_otel_span_from_kwargs)Logging)RouterBudgetLimiting)LeastBusyLoggingHandler)LowestCostLoggingHandler)LowestLatencyLoggingHandler)LowestTPMLoggingHandler)LowestTPMLoggingHandler_v2)simple_shuffle)get_deployments_for_tag)"_get_router_metadata_variable_namereplace_model_in_jsonl)InitalizeOpenAISDKClient)CooldownCache)DEFAULT_COOLDOWN_TIME_SECONDS_async_get_cooldown_deployments/_async_get_cooldown_deployments_with_debug_info_get_cooldown_deployments_set_cooldown_deployments)#_check_non_standard_fallback_formatget_fallback_model_grouprun_async_fallback)!get_num_retries_from_retry_policy)#async_raise_no_deployment_exceptionsend_llm_exception_alert)PromptCachingDeploymentCheck)0increment_deployment_failures_for_current_minute1increment_deployment_successes_for_current_minute)FlowItem	Scheduler)AllMessageValuesBatch
FileObject	FileTypes)#CONFIGURABLE_CLIENTSIDE_AUTH_PARAMSVALID_LITELLM_ENVIRONMENTSAlertingConfigAllowedFailsPolicyAssistantsTypedDictCustomRoutingStrategyBase
DeploymentDeploymentTypedDictGenericBudgetConfigTypeLiteLLM_ParamsModelGroupInfoOptionalPreCallChecksRetryPolicyRouterCacheEnumRouterGeneralSettingsRouterModelGroupAliasItemRouterRateLimitErrorRouterRateLimitErrorBasicRoutingStrategy)ServiceTypes)	ModelInfo)StandardLoggingPayload)CustomStreamWrapperEmbeddingResponseModelResponseget_llm_provider
get_secretget_utc_datetimeis_region_allowed   )PatternMatchRouter)Spanc                       \ rS rSrSrSrg)RoutingArgs   <    N)__name__
__module____qualname____firstlineno__ttl__static_attributes__r^       H/home/james-whalen/.local/lib/python3.13/site-packages/litellm/router.pyr[   r[      s    
Cre   r[   c            Q       P   \ rS rSr% / r\\S'   Sr\\	   \S'   Sr
\\S'   SrSr\\   \S'   Sr\\   \S	'   SSSSSSS0 SSSSSSSSSSS
S/ / / 0 SSSS0 SSSSSS0 SS\" 5       4'S\\\\   \\\\4      4      S\\   S\\   S\\   S\\   S\\   S\\	   S\S\\\      S\S\\   S\\   S\\   S\\   S\\   S\\   S\\   S\	S\S   S \\\      S!\S"\S#\S$\\\\\\4   4      S%\	S&\	S'\S(\\\\4      S)\\\4   S*\\   S+\\   S,\\   S-\\	   S.\S/   S0\\   S1\S2\\    S3\\!   S4\\   S5S4PS6 jjr"S7 r#S8\\   4S9 jr$S0\\   4S: jr%S.\\&\4   S1\4S; jr'S<\4S= jr(S>\S?\\\\4      S5\\)\*4   4S@ jr+S>\S?\\\\4      S5\\)\*4   4SA jr,\-S>\S?\\\\4      SB\SC   S5\*4SD j5       r.\- SS>\S?\\\\4      SB\S   S5\)4SE jj5       r.\- SS>\S?\\\\4      SB\\SC   \S   4   S5\\*\)4   4SF jj5       r. SS>\S?\\\\4      SB\	4SG jjr.S>\S?\\\\4      S5\\)\*4   4SH jr/S>\SI\S5S4SJ jr0SI\S5S4SK jr1S<\SI\S5S4SL jr2S<\SI\4SM jr3SI\SN\S5\\\\4      4SO jr4SP\\   S?\\\\\4      \\\\\4         4   4SQ jr5S>\S?\\\\\4         4SR jr6\-S>\S?\\\\4      SB\SC   S5\*4SS j5       r7\- SS>\S?\\\\4      SB\S   S5\)4ST jj5       r7 SS>\S?\\\\4      SB\	4SU jjr7\- SS>\S?\\\\4      SV\SB\S   S5\)4
SW jj5       r8\-S>\S?\\\\4      SV\SB\SC   S5\*4
SX j5       r8 SS>\S?\\\\4      SV\4SY jjr8SZ\S>\4S[ jr9SZ\S>\4S\ jr:SZ\S>\4S] jr;SZ\S>\4S^ jr<S_\=S>\4S` jr>S_\=S>\4Sa jr?S>\Sb\Sc\4Sd jr@S>\4Se jrAS>\4Sf jrBS>\4Sg jrC   SS>\SZ\Sh\\	   Si\\	   Sj\\	   4
Sk jjrD   SS>\SZ\Sh\\	   Si\\	   Sj\\	   4
Sl jjrES>\SZ\4Sm jrF   SSn\S>\Sh\\	   Si\\	   Sj\\	   4
So jjrGSn\S>\4Sp jrH SS>\Sb\\\4   Sj\\	   S5\I4Sq jjrJSb\\\4   S>\4Sr jrK SS>\Sb\\\4   Sj\\	   S5\I4Ss jjrLSb\\\4   S>\4St jrMS>\S5\N4Su jrOS>\S5\N4Sv jrPS>\S5\Q4Sw jrRS>\S5\Q4Sx jrSS5\Q4Sy jrTS>\4Sz jrUS{\V4S| jrW SS{\VS}\S~   4S jjrX  SS{\VS\\S      S\\Y   4S jjrZS r[    SSI\S\\   S!\\   S"\\   S#\\   4
S jjr\S r]S{\4S jr^ SSI\S\\   4S jjr_     SS\`S\\   S\\   S"\\   S#\\   S\\   4S jjraS rb SS!\\\\\   4      S\\   S5\\\      4S jjrc  SS\`S\S\S\\   S\\   S5\\\4   4S jjrdS reS5\\   4S jrfS5\	4S jrgS\\   4S jrhSI\S\`S5\4S jriS\S\\j   S5\4S jrk SS\S\\\4   S\\   S5\	4S jjrlS5\	4S jrmS>\S\)SI\S5\	4S jrnS>\S\\j   4S jroS>\S\\j   S5\p\\   \\   4   4S jrqS<\4S jrr SS<\S\\j   S\\s   4S jjrt  SS>\S\\   S?\\\u      S\\j   S\\   S\\s   4S jjrvS\S\4S jrwS\S\S\S\S5\\x   4
S jryS<\xS5\	4S jrzS\{4S jr|S<\xS5\x4S jr}S<\xS5\\x   4S jr~S<\xS5\\x   4S jrS\S5\\x   4S jrS\S5\\x   4S jrS\S5\\x   4S jr\- SS<\S\SSS5\4S jj5       r\-S<SS\S\S5\4S j5       r SS<\\   S\S\\   S5\4S jjrS\S5\\   4S jrS\S5\\   4S jrS\S\S5\\   4S jrS\S5\\   4S jrS\S5\p\\   \\   4   4S jrS\S5\\\4   4S jr SS\S\\   S5\4S jjrSS\\   S5\\   4S jjr SS\S\\   S5\\   4S jjrS5\\   4S jr SS\\   S5\\\      4S jjr SS\\   S5\\\\   4   4S jjrS rS rSS jr SS>\S\S?\\\\4      S\\   4S jjrS>\S5\\   4S jrS>\S5\4S jr   SS>\S?\\\\\4         Sb\\\\4      S\\	   S5\p\\\\4   4   4
S jjr    SS>\S?\\\\\4         Sb\\\\4      S\\	   S\\   4
S jjr    SS>\S?\\\\\4         Sb\\\\4      S\\	   S\\   4
S jjrS\\   S\\   S5\\   4S jr SS\\j   4S jjr SS\`S\\   4S jjrS\`4S jrS rS\4S jrS rS rSrg)Router   model_namesFcache_responsesi  default_cache_time_secondsNleastbusy_loggerlowesttpm_loggerINFOr   simple-shuffle
model_listassistants_config	redis_url
redis_host
redis_portredis_passwordcache_kwargscaching_groups
client_ttlpolling_intervaldefault_prioritynum_retriesmax_fallbackstimeoutdefault_litellm_paramsdefault_max_parallel_requestsset_verbosedebug_level)DEBUGro   default_fallbacks	fallbackscontext_window_fallbackscontent_policy_fallbacksmodel_group_aliasenable_pre_call_checksenable_tag_filteringretry_afterretry_policymodel_group_retry_policyallowed_failsallowed_fails_policycooldown_timedisable_cooldownsrouting_strategy)rp   
least-busyusage-based-routinglatency-based-routingcost-based-routingusage-based-routing-v2optional_pre_call_checksrouting_strategy_argsprovider_budget_configalerting_configrouter_general_settingsreturnc(                    SSK Jn(  UU l        UU l        UU l        UU l        U R                  SL aU  US:X  a%  [        R                  " [        R                  5        O*US:X  a$  [        R                  " [        R                  5        U'=(       d
    [        5       U l        X l        / U l        0 U l        Sn)Sn*0 n+Xl        Uc  UbJ  UbG  Sn)Ub  UU+S	'   Ub  UU+S
'   Ub  [#        U5      U+S'   Ub  UU+S'   U+R%                  U5        ['        S)0 U+D6n*U(       a8  [(        R*                  c!  [(        R,                  " S)SU)0U+D6[(        l        Xpl        [1        U*[3        5       S9U l        [5        UU*S9U l        Xl        SU l        UU l        / U l        [A        5       U l!        Ubb  [D        RF                  " U5      nU RI                  U5        U RJ                  U l&        U H#  n,SU,S   ;   d  M  SU R                  U,S   S   '   M%     O/ U l%        Ub  UU l'        O[(        RN                  U l'        U =(       d    [P        U l)        [U        U R*                  U RR                  S9U l+        U!U l,        [3        5       U l-        Ub  Xl.        O<[(        R\                  b  [(        R\                  U l.        O[^        R`                  U l.        Ub  Xl1        O<[(        Rb                  b  [(        Rb                  U l1        O[(        Rd                  U l1        U=(       d    [(        Rf                  U l4        UU l5        U"U l6        U=(       d    [(        Rn                  n-U Rq                  U-S9  U-U l7        Uc  [(        Rr                  bN  U=(       d    [(        Rr                  n-U Rn                  b  U Rn                  Ru                  SU-05        O
SU-0/U l7        U=(       d    [(        Rv                  U l;        U=(       d    [(        Rx                  n.U Rq                  U.S9  U.U l<        [{        [|        5      U l?        [{        [|        5      U l@        [{        [|        5      U lA        / U lB        U=(       d    0 U lC        U=(       d    0 n[(        R                  " UU S9U lE        UU lF        U R                  R                  SU5        U R                  R                  SS5        U R                  R                  S0 5      R%                  SU	05        0 U lH         U R                  U"U$S9  SU lJ        [        [(        R                  [        5      (       a*  [(        R                  Ru                  U R                  5        O)[(        R                  Ru                  U R                  5        [        [(        R                  [        5      (       a*  [(        R                  Ru                  U R                  5        OU R                  /[(        lO        [        [(        R                  [        5      (       a*  [(        R                  Ru                  U R                  5        OU R                  /[(        lQ        [        [(        R                  [        5      (       a*  [(        R                  Ru                  U R                  5        OU R                  /[(        lS        [        R                  " SU Rl                   SU R                   SU Rn                   SU Rx                   SU Rv                   S U R*                  R                   S!35        U(" 5       U lW        U$U lX        U%U lY        SU lZ        [        R                  " XR                  S"9(       a  U#b  U#Ru                  S#5        OS#/n#SU l]        Ub~  [        U[        5      (       a  [        S)0 UD6U l]        O[        U[        5      (       a  UU l]        [        R                  " S$R                  U R                  R                  SS%95      5        UU lc        SU ld        Ub~  [        U[        5      (       a  [        S)0 UD6U ld        O[        U[        5      (       a  UU ld        [        R                  " S&R                  U R                  R                  SS%95      5        U&U lf        U#b  U R                  U#5        U R                  b  U R                  5         U R                  5         U R                  [(        R                  S'S(9U lk        g)*a  
Initialize the Router class with the given parameters for caching, reliability, and routing strategy.

Args:
    model_list (Optional[list]): List of models to be used. Defaults to None.
    redis_url (Optional[str]): URL of the Redis server. Defaults to None.
    redis_host (Optional[str]): Hostname of the Redis server. Defaults to None.
    redis_port (Optional[int]): Port of the Redis server. Defaults to None.
    redis_password (Optional[str]): Password of the Redis server. Defaults to None.
    cache_responses (Optional[bool]): Flag to enable caching of responses. Defaults to False.
    cache_kwargs (dict): Additional kwargs to pass to RedisCache. Defaults to {}.
    caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None.
    client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600.
    polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms.
    default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None.
    num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2.
    timeout (Optional[float]): Timeout for requests. Defaults to None.
    default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}.
    set_verbose (bool): Flag to set verbose mode. Defaults to False.
    debug_level (Literal["DEBUG", "INFO"]): Debug level for logging. Defaults to "INFO".
    fallbacks (List): List of fallback options. Defaults to [].
    context_window_fallbacks (List): List of context window fallback options. Defaults to [].
    enable_pre_call_checks (boolean): Filter out deployments which are outside context window limits for a given prompt
    model_group_alias (Optional[dict]): Alias for model groups. Defaults to {}.
    retry_after (int): Minimum time to wait before retrying a failed request. Defaults to 0.
    allowed_fails (Optional[int]): Number of allowed fails before adding to cooldown. Defaults to None.
    cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1.
    routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle".
    routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}.
    alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None.
    provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None.
Returns:
    Router: An instance of the litellm.Router class.

Example Usage:
```python
from litellm import Router
model_list = [
{
    "model_name": "azure-gpt-3.5-turbo", # model alias
    "litellm_params": { # params for litellm completion/embedding call
        "model": "azure/<your-deployment-name-1>",
        "api_key": <your-api-key>,
        "api_version": <your-api-version>,
        "api_base": <your-api-base>
    },
},
{
    "model_name": "azure-gpt-3.5-turbo", # model alias
    "litellm_params": { # params for litellm completion/embedding call
        "model": "azure/<your-deployment-name-2>",
        "api_key": <your-api-key>,
        "api_version": <your-api-version>,
        "api_base": <your-api-base>
    },
},
{
    "model_name": "openai-gpt-3.5-turbo", # model alias
    "litellm_params": { # params for litellm completion/embedding call
        "model": "gpt-3.5-turbo",
        "api_key": <your-api-key>,
    },
]

router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
```
r   )ServiceLoggingTro   r   localNredisurlhostportpasswordtype)redis_cachein_memory_cache)rz   r   modellitellm_params)cachedefault_cooldown_time)fallback_param*)params
router_objr~   max_retriesmetadatarx   r   r   z)Intialized router with Routing strategy: z"

Routing enable_pre_call_checks: z

Routing fallbacks: z

Routing content fallbacks: z$

Routing context window fallbacks: z

Router Redis Caching=
)rq   r   router_budget_limitingz+[32mRouter Custom Retry Policy Set:
{}[0mexclude_nonez3[32mRouter Custom Allowed Fails Policy Set:
{}[0m
moderation)	call_typer^   )llitellm._service_loggerr   r   r   r   r   r   setLevelloggingro   r   rH   r   rr   deployment_namesdeployment_latency_mapry   strupdater   litellmr   Cacherk   r   r   r5   	schedulerr{   default_deploymentr   provider_default_deployment_idsrX   pattern_routercopydeepcopyset_model_listrq   healthy_deploymentsr   r&   r   r%   cooldown_cacher   failed_callsr|   openaiDEFAULT_MAX_RETRIESr}   ROUTER_MAX_FALLBACKSrequest_timeoutr~   r   r   r   validate_fallbacksr   appendr   r   r   inttotal_calls
fail_callssuccess_callsprevious_modelsr   Chatchatr   
setdefaultdeployment_statsrouting_strategy_initaccess_groups
isinstance_async_success_callbacklistdeployment_callback_on_successsuccess_callback#sync_deployment_callback_on_success_async_failure_callback$async_deployment_callback_on_failurefailure_callbackdeployment_callback_on_failuredebugr   service_logger_objr   r   router_budget_loggerr   !should_init_router_budget_limiterr   dictrF   infoformat
model_dumpr   r   r=   r   add_optional_pre_call_checks_initialize_alertinginitialize_assistants_endpointfactory_functionamoderation)/selfrq   rr   rs   rt   ru   rv   rk   rw   rx   ry   rz   r{   r|   r}   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
cache_typer   cache_configm
_fallbacks_content_policy_fallbackss/                                                  rf   __init__Router.__init__   s   b 	;&&&<#$8!t#f$%..w||<'%..w}}=#>'<'> 	$ "3 	 ')#  	 ')$ Z%;
@V J$&/U#%'1V$%'*:V$)+9Z( -$4|4K}}$ ' N: N N#2 #]_


 #-;
 !1"&-J*:<,02!z2J
+-1__D$a 011PQD//2B0CG0LM  
  O $!.D!(!6!6D*K.K+**D4F4F
 "3O 	 "*  ,&22D%99D$!."".!(!6!6D!(!=!=D9'"9"9& 0 3'"3"3
z:#(G,E,E,Q*Gg.G.GJ~~)%%sJ&78#&
"3!4 %H(H(H 	%
 %H(H(H 	" 	/HI(A%(3)
 (3(
 +6+
  	 # 	
 "8!=2LL(>4P	 '=###..y'B##..}a@##..z2>EE~.	
 ')	 	""-"7 	# 	
 "g55t<<++2243V3VW++2243V3VWg..55$$++D,T,TU(,(P(P'QG$g55t<<++2299
 99/G+ g..55$$++D,O,OP(,(K(K'LG$##78M8M7N O//3/J/J.K L""&..!1 2**.*G*G)H I1151N1N0O P$$(JJ$:$:#;2?	
 #1"2%:"&<#DH!AA!:U:U
 (3(//0HI,D+E(37#,--$/$?,$?!L+66$0!!&&DKK%%00d0C % 	% CG!+.55,>,VAU,V)02DEE,@)!&&LSS--88d8K :I#/--.FG+%%'++-00< 1 
re   c                 D   U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R
                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l	        g N)
r   r   acreate_assistantsadelete_assistantaget_assistantsacreate_threadaget_threada_add_messageaget_messagesarun_threadr   s    rf   r   %Router.initialize_assistants_endpoint/  s    "&"7"78R8R"S!%!6!6w7P7P!Q#44W5L5LM"33G4J4JK001D1DE!2273H3HI!2273H3HI001D1DEre   r   c           	          Uc  gU HQ  n[        U[        5      (       d  [        SU S35      e[        U5      S:w  d  M8  [        SU S[        U5       S35      e   g)z#
Validate the fallbacks parameter.
NzItem 'z' is not a dictionary.rW   zDictionary 'z%' must have exactly one key, but has z keys.)r   r   
ValueErrorlen)r   r   fallback_dicts      rf   r   Router.validate_fallbacks:  so     !+MmT22 6-8N!OPP=!Q& "=/1VWZ[hWiVjjpq 	 ,re   c                     Uby  U Hr  nS nUS:X  a  [        U R                  S9nO/US:X  a)  [        U R                  U R                  U R                  S9nUc  MS  [
        R                  R                  U5        Mt     g g )Nprompt_caching)r   r   )
dual_cacher   rq   )r1   r   r   r   rq   r   	callbacksr   )r   r   pre_call_check	_callbacks       rf   r   #Router.add_optional_pre_call_checksI  s{     $/":48	!%55 <4:: NI#'?? 4#'::/3/J/J#'??!I
 (%%,,Y7 #; 0re   c                 b   [         R                  " SU 35        U[        R                  R                  :X  d  U[        R                  :X  a  [        U R                  U R                  S9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        OU R                  /[        l        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        g g U[        R                  R                  :X  d  U[        R                  :X  ar  [!        U R                  U R                  US9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R"                  5        g g U[        R$                  R                  :X  d  U[        R$                  :X  ar  ['        U R                  U R                  US9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R(                  5        g g U[        R*                  R                  :X  d  U[        R*                  :X  ar  [-        U R                  U R                  US9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R.                  5        g g U[        R0                  R                  :X  d  U[        R0                  :X  ar  [3        U R                  U R                  0 S9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R4                  5        g g g )NzRouting strategy: )router_cacherq   )r  rq   routing_args)r   r   rL   
LEAST_BUSYvaluer   r   rq   rm   r   r   input_callbackr   r   r  USAGE_BASED_ROUTINGr   rn   USAGE_BASED_ROUTING_V2r   lowesttpm_logger_v2LATENCY_BASEDr   lowestlatency_logger
COST_BASEDr   lowestcost_logger)r   r   r   s      rf   r   Router.routing_strategy_initZ  s    	""%78H7I#JK : : @ @@?#=#==$;!ZZDOO%D! '00$77&&--d.C.CD*.*?*?)@&'++T22!!(()>)>? 3  C C I II?#F#FF$;!ZZ??2%D!
 '++T22!!(()>)>? 3  F F L LL?#I#II'A!ZZ??2(D$
 '++T22!!(()A)AB 3  = = C CC?#@#@@(C!ZZ??2)D%
 '++T22!!(()B)BC 3  : : @ @@?#=#==%=!ZZ??&D"
 '++T22!!(()?)?@ 3 re   
deploymentc                      [         R                  " U5      nUS   nSU;   a  US   SS S-   US'   U$ ! [         a)  n[        R                  " S[        U5       35        UeSnAff = f)z
returns a copy of the deployment with the api key masked

Only returns 2 characters of the api key and masks the rest with * (10 *).
r   api_keyN   z
**********z+Error occurred while printing deployment - )r   r   	Exceptionr   r   r   )r   r#  _deployment_copyr   es        rf   print_deploymentRouter.print_deployment  s}    
	#}}Z8#34D#ENN*,:9,Ebq,IH,Ty)## 	!''=c!fXF G		s   03 
A&$A!!A&r   messagesc                      [         R                  " SU S35        XS'   X#S'   U R                  US'   U R                  XS9  U R                  " S0 UD6nU$ ! [
         a  nUeSnAff = f)	z
Example usage:
response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}]
zrouter.completion(model=z,..)r   r,  original_functionr   kwargsNr^   )r   r   _completion_update_kwargs_before_fallbacksfunction_with_fallbacksr'  )r   r   r,  r0  responser)  s         rf   
completionRouter.completion  s}    
	!''*B5'(NO#7O!):*.*:*:F&'00u0L33=f=HO 	G	s   AA 
A' A""A'c           	         S n U R                  UUUR                  SS 5      S9nU R                  XSS9  US   R                  5       nUS   nU R	                  XSS9nUR                  SS 5      nUb  Ub  XR                  :w  a  S n	OUn	XR                  5       ;  a  U R                  US9  [        R                  " S0 0 UEUU R                  U	S.EUED6n
[        R                  " S	U S
35        [        U
[        5      (       a-  U R!                  XUS9nU(       a  [        R"                  " SUSS9eU
$ ! [$         a-  n[        R                  " S	U S['        U5       S35        UeS nAff = f)Nspecific_deploymentr   r,  r8  r#  r0  r   r   r%  r#  r,  cachingclientzlitellm.completion(model=)[32m 200 OK[0mr   r4  r0  Response output was blocked. messager   llm_provider)[31m Exception [0mr^   )get_available_deploymentpop_update_kwargs_with_deploymentr   _get_clientgetr%  get_model_ids routing_strategy_pre_call_checksr   r5  rk   r   r   r   rR   "_should_raise_content_policy_errorContentPolicyViolationErrorr'  r   )r   r   r,  r0  
model_namer#  datapotential_model_clientdynamic_api_keymodel_clientr4  _should_raiser)  s                rf   r1  Router._completion  s    
<	66!$*JJ/Dd$K 7 J
 //:/U./446DgJ%)%5%5% &6 &" %jjD9O+*6#'E'EE#5 ..00555L))  (#33*	
 H "&&+J<7NO
 (M22 $ G G6 !H ! !!== >#%'  O 	!&&+J<7KCPQF8SZ[ G		s   D,D1 1
E(;(E##E(streamTc                    #    g 7fr   r^   r   r   r,  rX  r0  s        rf   acompletionRouter.acompletion  
      	   c                    #    g 7fr   r^   rZ  s        rf   r[  r\    r]  r^  c                    #    g 7fr   r^   rZ  s        rf   r[  r\    r]  r^  c                   #     XS'   X$S'   X4S'   U R                   US'   U R                  XS9  UR                  S5      =(       d    U R                  n[        R                  " 5       nUb0  [        U[        5      (       a  U R                  " S
0 UD6I S h  vN nOU R                  " S
0 UD6I S h  vN n[        R                  " 5       nX-
  n	[        R                  " U R                  R                  [        R                  U	SUU[        U5      S95        U$  N Nj! [          a:  n
[        R                  " [#        U U[$        R&                  " 5       U
S	95        U
eS n
A
ff = f7f)Nr   r,  rX  r.  r/  priorityr[  servicedurationr   
start_timeend_timeparent_otel_spanlitellm_router_instancerequest_kwargserror_traceback_stroriginal_exceptionr^   )_acompletionr2  rL  r{   timer   r   schedule_acompletionasync_function_with_fallbacksasynciocreate_taskr   async_service_success_hookrM   ROUTERr   r'  r0   	traceback
format_exc)r   r   r,  rX  r0  request_priorityrf  r4  rg  	_durationr)  s              rf   r[  r\    sR    %	#7O!):%8*.*;*;F&'00u0L%zz*5N9N9NJ+
;KS0Q0Q!%!:!:!DV!DD!%!C!C!Mf!MMyy{H -I''BB(//&+)%%Fv%N C 	 O! EM  		(,0#)(1(<(<(>'(	 G		sN   E#BD DD /D0A'D E#D D 
E &5EE  E#c                   #    Sn [         R                  " SU SU 35        [        U5      n[        R                  " 5       nU R	                  UUUR                  SS5      US9I Sh  vN n[        R                  " 5       nX-
  n	[        R                  " U R                  R                  [        R                  U	SUU[        U5      S95        U R                  XuS9  U R                  XsS	9  US
   R                  5       n
U
S   nU R                  UUS	9nU R                   U==   S-  ss'   ["        R$                  " S0 0 U
EUU R&                  US.EUED6nUR)                  SS5      nU R+                  UUSS9nUb_  [-        U[        R.                  5      (       a@  U ISh  vN    U R1                  UUUS9I Sh  vN   UI Sh  vN nSSS5      ISh  vN   O#U R1                  UUUS9I Sh  vN   UI Sh  vN n[-        W[2        5      (       a-  U R5                  XUS9nU(       a  ["        R6                  " SUSS9eU R8                  U==   S-  ss'   [         R:                  " SU S35        U R                  UUUS9  U$  GN N N N N! , ISh  vN  (       d  f       N= f N N! [<         aG  n[         R:                  " SU S[?        U5       S35        Ub  U R@                  U==   S-  ss'   UeSnAff = f7f)z
- Get an available deployment
- call it with a semaphore over the call
- semaphore specific to it's rpm
- in the semaphore,  make a check against it's local rpm before running
NzInside _acompletion()- model: 
; kwargs: r8  )r   r,  r8  rk  async_get_available_deploymentrc  r#  rh  r:  r   r   rW   r<  litellm_logging_objmax_parallel_requestsr#  r0  client_type)r#  logging_objrh  r@  rA  rB  rC  zlitellm.acompletion(model=r?  )r#  r4  rh  rF  rG  r^   )!r   r   r   ro  r|  rI  rr  rs  r   rt  rM   ru  _track_deployment_metricsrJ  r   _get_async_openai_model_clientr   r   r[  rk   rL  rK  r   	Semaphore&async_routing_strategy_pre_call_checksrR   rO  rP  r   r   r'  r   r   )r   r   r,  r0  rQ  rh  rf  r#  rg  ry  rR  rU  	_responser  rpm_semaphorer4  rV  r)  s                     rf   rn  Router._acompletionE  s     
q	!''0z&J  AHJ#BB!$*JJ/Dd$K%	  C   J yy{H -I''BB(//&>)%%Fv%N C 	 **% +  //:/U./446DgJ>>% ? L Z(A-(++  (#33*	
 I 5;JJ%t5K !,,%3 - M
 (Zw00. . )= EE#-$/)9 F   
 &/H )== AA) +%5 B    "+? (M22 $ G G6 !H ! !!== >#%'  z*a/*!&&,ZL8OP **%!!1 +  OIr )

  / )=== +4  	!&&,ZL8LSQRVHT[\ %
+q0+G	s   L A!J, 'J(D8J,  J!J, $J:J;	JJ
J	J, JJ, .J(/	J, 8J*9B	J, L J, J, J
JJ, J%JJ%!J, *J, ,
K=6AK88K==L r0  c                     UR                  SU R                  5      US'   UR                  S[        [        R
                  " 5       5      5        UR                  S0 5      R                  SU05        g)zE
Adds/updates to kwargs:
- num_retries
- litellm_trace_id
- metadata
r|   litellm_trace_idr   model_groupN)rL  r|   r   r   uuiduuid4r   )r   r   r0  s      rf   r2  &Router._update_kwargs_before_fallbacks  s[     !'

=$:J:J K},c$**,.?@*b)00-1GHre   c                     U R                   R                  5        H.  u  p#X!;  a	  Ub  X1U'   M  US:X  d  M  X   R                  U5        M0     g)z0
Adds default litellm params to kwargs, if set.
Nr   )r   itemsr   )r   r0  kvs       rf   *_update_kwargs_with_default_litellm_params1Router._update_kwargs_with_default_litellm_params  sF     //557DAAMq	j	  # 8re   c                 *   UR                  S0 5      R                  US   S   UR                  S0 5      UR                  S0 5      R                  S5      S.5        UR                  S0 5      US'   U R                  X!S   S9US'   U R	                  US	9  g
)z
2 jobs:
- Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
- Adds default litellm params to kwargs, if set.
r   r   r   
model_infoapi_baser#  r  r  )r0  rR  r~   )r0  N)r   r   rL  _get_timeoutr  )r   r#  r0  s      rf   rJ  %Router._update_kwargs_with_deployment  s     	*b)00()9:7C(nn\2>&NN+;R@DDZP	
  *~~lB?| --+; < . 
y 	77v7Fre   c                     U R                  XSS9nUR                  SS5      nUb  Ub  XCR                  :w  a  SnU$ UnU$ )aY  
Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment

The same OpenAI client is re-used to optimize latency / performance in production

If dynamic api key is provided:
    Do not re-use the client. Pass model_client=None. The OpenAI/ AzureOpenAI client will be recreated in the handler for the llm provider
asyncr  r%  N)rK  rL  r%  )r   r#  r0  rS  rT  rU  s         rf   r  %Router._get_async_openai_model_client  sc     "&!1!1!g "2 "

 !**Y5'&2#A#AAL  2Lre   rR  c                 ,   UR                  SS5      =(       dz    UR                  SS5      =(       da    UR                  SS5      =(       dH    UR                  SS5      =(       d/    U R                  =(       d    U R                  R                  SS5      nU$ )z6Helper to get timeout from kwargs or deployment paramsr~   Nr   )rL  r~   r   )r   r0  rR  r~   s       rf   r  Router._get_timeout  s     JJy$' 	@zz+T2	@xx4	@
 xx!4	@ ||	@ **..y$? 	 re   modelsc                   ^ #    S[         S[        [        [         [         4      4U 4S jjnS[         S[        [        [         [         4      S[        4U 4S jjn[	        U[
        5      (       aX  [        S U 5       5      (       aA  / nU H  nUR                  U" SXrS.UD65        M     [        R                  " U6 I Sh  vN nU$ [	        U[
        5      (       a  [        S	 U 5       5      (       a  / n[        U5       H(  u  pU H  nUR                  U" SXyU
S
.UD65        M     M*     [        R                  " U6 I Sh  vN n[        [        U5      5       Vs/ s H  n/ PM     nnU HG  n[	        U[        5      (       a  XS      R                  US   5        M3  US   R                  U5        MI     U$ gg N N{s  snf 7f)av  
Async Batch Completion. Used for 2 scenarios:
1. Batch Process 1 request to N models on litellm.Router. Pass messages as List[Dict[str, str]] to use this
2. Batch Process N requests to M models on litellm.Router. Pass messages as List[List[Dict[str, str]]] to use this

Example Request for 1 request to N models:
```
    response = await router.abatch_completion(
        models=["gpt-3.5-turbo", "groq-llama"],
        messages=[
            {"role": "user", "content": "is litellm becoming a better product ?"}
        ],
        max_tokens=15,
    )
```


Example Request for N requests to M models:
```
    response = await router.abatch_completion(
        models=["gpt-3.5-turbo", "groq-llama"],
        messages=[
            [{"role": "user", "content": "is litellm becoming a better product ?"}],
            [{"role": "user", "content": "who is this"}],
        ],
    )
```
r   r,  c                 |   >#     TR                   " SXS.UD6I Sh  vN $  N! [         a  nUs SnA$ SnAff = f7fz[
Wrapper around self.async_completion that catches exceptions and returns them as a result
r   r,  Nr^   r[  r'  r   r,  r0  r)  r   s       rf   _async_completion_no_exceptionsARouter.abatch_completion.<locals>._async_completion_no_exceptions<  ?     !--WEWPVWWWW 0   <# !# <# 
949<9<idxc                    >#     TR                   " SXS.UD6I Sh  vN U4$  N! [         a  nXB4s SnA$ SnAff = f7fr  r  )r   r,  r  r0  r)  r   s        rf   *_async_completion_no_exceptions_return_idxLRouter.abatch_completion.<locals>._async_completion_no_exceptions_return_idxG  sL     **TTVTT T  vs0   ?% #% ?% 
<7<?<?c              3   B   #    U  H  n[        U[        5      v   M     g 7fr   )r   r   .0r   s     rf   	<genexpr>+Router.abatch_completion.<locals>.<genexpr>Z  s     -T8ajD.A.A8   r  Nc              3   B   #    U  H  n[        U[        5      v   M     g 7fr   )r   r   r  s     rf   r  r  a  s     /VX
1d0C0CXr  )r   r  r,  rW   r   r^   )r   r	   r   r   r   r   allr   rr  gather	enumerateranger  tuple)r   r  r,  r0  r  r  _tasksr   r4  r  rD  	responses_final_responsess   `             rf   abatch_completionRouter.abatch_completion  s    H				"&tCH~"6				4S>*	 	& h%%#-T8-T*T*TF=gEg`fgh   %^^V44HO$''C/VX/V,V,VF )( 3#EMMB "'7FL $ !4 &nnf55I<A#h-<P/Q<Pq<PO/Q%h..#QK077D#A&--h7	 &
 #"# -W' 5 6/Qs8   B=G F<BGF>GG )AG>G Gc           	         ^ #    S[         S[        [        [         [         4      4U 4S jjn/ nU H  nUR                  U" SXS.UD65        M     [        R
                  " U6 I Sh  vN nU$  N7f)a  
Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router

Use this for sending multiple requests to 1 model

Args:
    model (List[str]): model group
    messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
    **kwargs: additional kwargs
Usage:
    response = await self.abatch_completion_one_model_multiple_requests(
        model="gpt-3.5-turbo",
        messages=[
            [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
            [{"role": "user", "content": "hello good mornign"}],
        ]
    )
r   r,  c                 |   >#     TR                   " SXS.UD6I Sh  vN $  N! [         a  nUs SnA$ SnAff = f7fr  r  r  s       rf   r  ]Router.abatch_completion_one_model_multiple_requests.<locals>._async_completion_no_exceptions  r  r  r  Nr^   )r   r	   r   r   rr  r  )r   r   r,  r0  r  r  message_requestr4  s   `       rf   -abatch_completion_one_model_multiple_requests4Router.abatch_completion_one_model_multiple_requestst  sz     ,				"&tCH~"6		 'OMM/ =C  ( !00 1s   A#A/&A-'A/c                    #    g 7fr   r^   rZ  s        rf   "abatch_completion_fastest_response)Router.abatch_completion_fastest_response  r]  r^  c                    #    g 7fr   r^   rZ  s        rf   r  r    r]  r^  c                   ^ ^#    UR                  S5       Vs/ s H  oUR                  5       PM     nnS[        S[        [        [        [        4      S[
        S[        S[        [        [        [        4   4
U 4S jjn/ mS[        R                  4U4S	 jjnU H3  n[        R                  " U" SXUS
.UD65      n	TR                  U	5        M5     T(       af  [        R                  " T[        R                   S9I Sh  vN u  n
mU
 H)  nU" U5      I Sh  vN nUc  M  SUR"                  S'   Us  $    T(       a  Mf  [        S5      es  snf  NO N97f)z
model - List of comma-separated model names. E.g. model="gpt-4, gpt-3.5-turbo"

Returns fastest response from list of model names. OpenAI-compatible endpoint.
,r   r,  rX  r0  r   c                    >#     TR                   " SXUS.UD6I Sh  vN $  N! [        R                   a'    [        R                  " SR                  U 5      5        e [         a  nUs SnA$ SnAff = f7f)zV
Wrapper around self.acompletion that catches exceptions and returns them as a result
r   r,  rX  Nz4Received 'task.cancel'. Cancelling call w/ model={}.r^   )r[  rr  CancelledErrorr   r   r   r'  )r   r,  rX  r0  r)  r   s        rf   r  RRouter.abatch_completion_fastest_response.<locals>._async_completion_no_exceptions  sr     !--fEU[f_effff)) %++JQQRWX  s>   A6$ "$ A6$ AA3'A.(A3)A6.A33A6taskc                   >#     U I S h  vN n[        U[        [        45      (       aC  [        R                  " S5        T H  nUR                  5         M     U TR                  U 5        $   TR                  U 5        g  Nv! [         a     $ f = f! [         a     N1f = f! [         a     g f = f!  TR                  U 5        f ! [         a     f f = f= f7f)Nz=Received successful response. Cancelling other LLM API calls.)	r   rR   rP   r   r   cancelremoveKeyErrorr'  )r  resulttpending_taskss      rf   check_responseARouter.abatch_completion_fastest_response.<locals>.check_response  s     #f}6I&JKK)//W +
 +!!((. L!((. $         !((. s   CB BAB B,C-B3 /B#  CB 
BCBC
B B3 B  B3 #
B0-C/B00C3C5CC
CCCCCr  )return_whenNT!fastest_response_batch_completionzAll tasks failedr^   )splitstripr   r	   r   boolr   r   rR   rP   r'  rr  Taskrs  r   waitFIRST_COMPLETED_hidden_params)r   r   r,  rX  r0  r   r  r  r  r  donecompleted_taskr  r  s   `            @rf   r  r    sR     &+[[%56%5'')%56		"&tCH~"6	@D	PS	="5y@A	  	w|| 	, E&&/ 6EKD
   &  (/7+B+B) #D- #'-n==%QUF))*MN!M #'	 m *++ 7h# >s:   EE
CE?E EEE!E?EErb  c                    #    g 7fr   r^   r   r   r,  rb  rX  r0  s         rf   rp  Router.schedule_acompletion  r]  r^  c                    #    g 7fr   r^   r  s         rf   rp  r  
  r]  r^  c                   #    [        U5      n[        [        R                  " 5       5      n[	        UUSS9nU R
                  R                  US9I S h  vN   [        R                  " 5       U R                  -   n	[        R                  " 5       n
U R
                  R                  nSnX:  a  U R                  XS9I S h  vN u  pU R
                  R                  UR                  UR                  US9I S h  vN nU(       a  O:[        R                  " U5      I S h  vN   [        R                  " 5       n
X:  a  M  U(       a\   U R                   " SXUS.UD6I S h  vN nUR"                  R%                  S0 5        UR"                  S   R'                  S	S
05        U$ [,        R.                  " SUSS9e GNd N N N Na! [(         a  n[+        USU5        UeS nAff = f7f)Nzgpt-3.5-turbo)rb  
request_idrQ  )requestFr   rh  )idrQ  health_deploymentsr  additional_headersz%x-litellm-request-prioritization-usedTrb  z%Request timed out while polling queuer   rC  r^   )r   r   r  r  r4   r   add_requestro  r~   rz   _async_get_healthy_deploymentspollr  rQ  rr  sleepr[  r  r   r   r'  setattrr   Timeout)r   r   r,  rb  rX  r0  rh  _request_iditemrg  	curr_timepoll_intervalmake_request_healthy_deploymentsr  r  r)  s                    rf   rp  r    s     =VD$**,'"&
 nn(((666 99;-IIK	77",0,O,O -P - '#  "&!4!4????#7 "5 " L
 mmM222 IIK	 " "&"2"2 #6#EK# 	 ((334H"M(()=>EE<dC ! 
 //?% K 	7' 3
  :x0s   AG#F7A*G#>F:?8G#7F<8%G#F>G#>G#G G AG  G#:G#<G#>G# G 
G GG  G#promptc                     X#S'   XS'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6nU$ ! [         a  nUeS nAff = f)Nr   r  r.  r|   r   r  r^   )_image_generationrL  r|   r   r   r3  r'  r   r  r   r0  r4  r)  s         rf   image_generationRouter.image_generationP  s    
	#7O%8*.*@*@F&'$*JJ}d>N>N$OF=!j"-44mU5KL33=f=HO 	G	   A,A/ /
B 9A;;B c           	         Sn [         R                  " SU SU 35        U R                  USSS./UR                  SS 5      S9nU R	                  XSS	9  US
   R                  5       nU R                  UUS	9nU R                  U==   S-  ss'   U R                  US9  [        R                  " S0 0 UEUU R                  US.EUED6nU R                  U==   S-  ss'   [         R                  " SU S35        U$ ! [         aG  n	[         R                  " SU S[        U	5       S35        Ub  U R                   U==   S-  ss'   U	eS n	A	ff = f)NrB  #Inside _image_generation()- model: r{  userr  rolecontentr8  r9  r:  r   rW   r;  r  r=  r>  zlitellm.image_generation(model=r?  rF  rG  r^   )r   r   rH  rI  rJ  r   r  r   rN  r   r  rk   r   r   r'  r   r   )
r   r  r   r0  rQ  r#  rR  rU  r4  r)  s
             rf   r   Router._image_generation]  s   
*	!''5eWJvhO 66#)h?@$*JJ/Dd$K 7 J
 //:/U./446D>>% ? L
 Z(A-( 11Z1H// $#33*	
 H z*a/*!&&1*=TU O 	!&&1*=QRUVWRXQYY`a %
+q0+G	s   C5C: :
EAEEc           
        #     X#S'   XS'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)	Nr   r  r.  r|   r   r  ri  r^   )_aimage_generationrL  r|   r   r   rq  r'  rr  rs  r0   rv  rw  r  s         rf   aimage_generationRouter.aimage_generation  s     	#7O%8*.*A*AF&'$*JJ}d>N>N$OF=!j"-44mU5KL!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   CA.A; 2A93A; 8C9A; ;
B?5B::B??Cc           	      ,  #    Un [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      S9I S h  vN nU R                  XcS9  US	   R                  5       nUS
   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XeS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XeS9I S h  vN   U	I S h  vN n	U R"                  U==   S-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S-  ss'   UeS nAff = f7f)Nr  r{  r  r  r  r8  r9  r:  r   r   rW   r  r  r  r}  z litellm.aimage_generation(model=r?  rF  rG  r^   )r   r   r   r|  rI  rJ  r   r  r   r   r  rk   rK  r   rr  r  r  r   r   r'  r   r   )r   r  r   r0  rQ  rh  r#  rR  rU  r4  r  r)  s               rf   r  Router._aimage_generation  s    
C	!''5eWJvhO  AH#BB#)h?@$*JJ/Dd$K  C   J
 //:/U./446DgJ>>% ? L
 Z(A-(00 $#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&2:,>UV OoH )
  . )=== *  	!&&2:,>RSVWXSYRZZab %
+q0+G	s   HAG  FB5G  FG  F"#F$	F"-F.F"2G  =F >G  F<	G  F> 6G  HG  G  F"F" G  "F9(F+)F95G  >G   
H
AHHHfilec           
      "  #     X#S'   XS'   U R                   US'   U R                  X#S9  U R                  " S0 UD6I Sh  vN nU$  N! [         a:  n[        R
                  " [        U U[        R                  " 5       US95        UeSnAff = f7f)aB  
Example Usage:

```
from litellm import Router
client = Router(model_list = [
    {
        "model_name": "whisper",
        "litellm_params": {
            "model": "whisper-1",
        },
    },
])

audio_file = open("speech.mp3", "rb")
transcript = await client.atranscription(
model="whisper",
file=audio_file
)

```
r   r  r.  r/  Nri  r^   )	_atranscriptionr2  rq  r'  rr  rs  r0   rv  rw  )r   r  r   r0  r4  r)  s         rf   atranscriptionRouter.atranscription  s     .	#7O!6N*.*>*>F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		9   B;A A A BA 
B5BBBc           	      "  #    Un [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      S9I S h  vN nU R                  XcS9  US	   R                  5       nU R                  UUS9nU R                  U==   S
-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XeS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XeS9I S h  vN   U	I S h  vN n	U R"                  U==   S
-  ss'   [         R$                  " SU S35        U	$  GN= N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S
-  ss'   UeS nAff = f7f)Nz!Inside _atranscription()- model: r{  r  r  r  r8  r9  r:  r   rW   )r  r=  r>  r  r  r}  zlitellm.atranscription(model=r?  rF  rG  r^   )r   r   r   r|  rI  rJ  r   r  r   r   r  rk   rK  r   rr  r  r  r   r   r'  r   r   )r   r  r   r0  rQ  rh  r#  rR  rU  r4  r  r)  s               rf   r  Router._atranscription  sy    
A	!''3E7*VHM  AH#BB#)h?@$*JJ/Dd$K  C   J //:/U./446D>>% ? L
 Z(A-(--  #33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&/
|;RS OkD )
  . )=== *  	!&&/
|;OPSTUPVxW^_ %
+q0+G	s   HAF; FB0F; FF; 
FF	F(F)F-F; 8F9F; F7	F; F96F; HF; F; FFF; F4#F&$F40F; 9F; ;
HAHHHinputvoicec           
      6  #     X$S'   X4S'   U R                  USSS./UR                  SS5      S9I Sh  vN nUR                  S	0 5      R                  US
   S   UR	                  S0 5      S.5        UR	                  S0 5      US'   US
   R                  5       nUS     U R                  R                  5        H+  u  pxXt;  a  XU'   M  US	:X  d  M  XG   R                  U5        M-     U R                  XTSS9n	UR	                  SS5      n
U
b  U	b  XR                  :w  a  SnOU	n[        R                  " S0 0 UESU0EUED6I Sh  vN nU$  GN N	! [         a:  n[        R                  " [        U U[         R"                  " 5       US95        UeSnAff = f7f)a  
Example Usage:

```
from litellm import Router
client = Router(model_list = [
    {
        "model_name": "tts",
        "litellm_params": {
            "model": "tts-1",
        },
    },
])

async with client.aspeech(
    model="tts",
    voice="alloy",
    input="the quick brown fox jumped over the lazy dogs",
    api_base=None,
    api_key=None,
    organization=None,
    project=None,
    max_retries=1,
    timeout=600,
    client=None,
    optional_params={},
) as response:
    response.stream_to_file(speech_file_path)

```
r  r  r  r  r  r8  Nr9  r   r   r   r  )r#  r  r  r  r%  r>  ri  r^   )r|  rI  r   r   rL  r   r   r  rK  r%  r   aspeechr'  rr  rs  r0   rv  rw  )r   r   r  r  r0  r#  rR  r  r  rS  rT  rU  r4  r)  s                 rf   r  Router.aspeechV  s    @9	#7O#7O#BB#)h?@$*JJ/Dd$K  C   J
 j"-44",-=">w"G",..r"B $.>>,#CF< ./446DM3399;O !1I*_I$$Q' < &*%5%5%' &6 &" %jjD9O+*6#'E'EE#5$__ l  H OWH  		(,0#)(1(<(<(>'(	 G		sR   F0E EBE A4E EE FE E 
F5FFFc           
      ,  #     XS'   [         US'   U R                  US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  N! [         a:  n[
        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7fNr   r  r.  r/  ri  r^   )
r  _arerankr2  rq  r'  rr  rs  r0   rv  rw  r   r   r0  r4  r)  s        rf   arerankRouter.arerank  s     	#7O#F7O*.--F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   BA A AA 
BA 
B5BBBc           	        #    S n [         R                  " SU SU 35        U R                  UUR                  SS 5      S9I S h  vN nU R	                  XBS9  US   R                  5       nUS   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEU R                  US	.EUED6I S h  vN nU R                  U==   S-  ss'   [         R                  " S
U S35        U$  N N9! [         aG  n[         R                  " S
U S[        U5       S35        Ub  U R                  U==   S-  ss'   UeS nAff = f7f)NzInside _rerank()- model: r{  r8  )r   r8  r:  r   r   rW   )r=  r>  zlitellm.arerank(model=r?  rF  rG  r^   )r   r   r|  rI  rJ  r   r  r   r   r%  rk   r   r   r'  r   r   )	r   r   r0  rQ  r#  rR  rU  r4  r)  s	            rf   r#  Router._arerank  s    
&	!''+E7*VHE  $BB$*JJ/Dd$K  C   J //:/U./446DgJ>>% ? L Z(A-($__ #33* 	 H z*a/*!&&(4KL O7  	!&&(4HQPWX %
+q0+G	sN   E?D  C<A>D  C>6D  ;E<D  >D   
E
AEEEc                   #    SSS./n UR                  SU R                  5      US'   UR                  S0 5      R                  SU05        U R	                  UUUR                  SS 5      S9I S h  vN nUS	   R                  5       nU R                  R                  5        H+  u  pgXb;  a  XrU'   M  US:X  d  M  X&   R                  U5        M-     [        R                  " S0 0 UES
U R                  0EUED6I S h  vN $  N N! [         aN  nU R                  S:  a7  XS'   X2S'   U R                  US'   U R                  " S0 UD6I S h  vN  s S nA$ UeS nAff = f7f)Nr  z
dummy-textr  r|   r   r  r8  r9  r   r=  r   r   r,  r.  r^   )rL  r|   r   r   r|  rI  r   r   r  r   
_arealtimerk   r'  async_function_with_retries)	r   r   r0  r,  r#  rR  r  r  r)  s	            rf   r*  Router._arealtime  so    #=>	$*JJ}d>N>N$OF=!j"-44mU5KL  $BB!$*JJ/Dd$K  C   J ./446D3399;O !1I*_I$$Q' < !++b.a.ay$BVBV.aZ`.abbb c 	!#"'w%-z".2oo*+!==GGGGG	ss   E A&D 0D1AD :AD <D=D  E D D 
E<EEEEE EEE is_retryis_fallbackis_asyncc                    SUS./n XS'   X&S'   UR                  SU R                  5      US'   UR                  S0 5      R                  SU05        U R	                  UUUR                  SS 5      S	9nUS
   R                  5       n	U R                  R                  5        H+  u  pX;  a  XU
'   M  U
S:X  d  M  Xj   R                  U5        M-     [        R                  " S0 0 U	EX R                  S.EUED6$ ! [         a  nUeS nAff = f)Nr  r  r   r  r|   r   r  r8  r9  r   )r  r=  r^   )rL  r|   r   r   rH  rI  r   r   r  r   text_completionrk   r'  )r   r   r  r-  r.  r/  r0  r,  r#  rR  r  r  r)  s                rf   r1  Router.text_completion  s"    $78	#7O%8$*JJ}d>N>N$OF=!j"-44mU5KL 66!$*JJ/Dd$K 7 J ./446D3399;O !1I*_I$$Q' < **s-r-rSgSg-rkq-rss 	G	s   B,C7 8>C7 7
DDDc           
      "  #     XS'   X&S'   U R                   US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R
                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)Nr   r  r.  r/  ri  r^   )	_atext_completionr2  rq  r'  rr  rs  r0   rv  rw  )	r   r   r  r-  r.  r/  r0  r4  r)  s	            rf   atext_completionRouter.atext_completion5  s     	#7O%8*.*@*@F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		r  c           	      (  #     [         R                  " SU SU 35        [        U5      nU R                  USUS./UR	                  SS 5      S9I S h  vN nU R                  XSS9  US   R                  5       nUS	   nU R                  UUS9nU R                  U==   S
-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XTS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XTS9I S h  vN   U	I S h  vN n	U R"                  U==   S
-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S
-  ss'   UeS nAff = f7f)N#Inside _atext_completion()- model: r{  r  r  r8  r9  r:  r   r   rW   r  r  r  r}  zlitellm.atext_completion(model=r?  rF  rG  r^   )r   r   r   r|  rI  rJ  r   r  r   r   r5  rk   rK  r   rr  r  r  r   r   r'  r   r   )r   r   r  r0  rh  r#  rR  rQ  rU  r4  r  r)  s               rf   r4  Router._atext_completionQ  s{    B	!''5eWJvhO  AH#BB#)f=>$*JJ/Dd$K  C   J
 //:/U./446DgJ>>% ? L Z(A-(// $#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&1*=TU OmF )
  . )=== *  	!&&1%8LSQRVHT[\  &!+&G	   HAF> FB5F> 	F
F> F !F"	F +F,F 0F> ;F<F> F:	F> F<6F> HF> F> F F F>  F7&F)'F73F> <F> >
HAH

HH
adapter_idc           
        #     X&S'   XS'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)	Nr   r;  r.  r|   r   r  ri  r^   )_aadapter_completionrL  r|   r   r   rq  r'  rr  rs  r0   rv  rw  )	r   r;  r   r-  r.  r/  r0  r4  r)  s	            rf   aadapter_completionRouter.aadapter_completion  s     	#7O#-< *.*C*CF&'$*JJ}d>N>N$OF=!j"-44mU5KL!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		r  c           	      (  #     [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      S9I S h  vN nU R                  XSS9  US	   R                  5       nUS
   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XTS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XTS9I S h  vN   U	I S h  vN n	U R"                  U==   S-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S-  ss'   UeS nAff = f7f)Nz&Inside _aadapter_completion()- model: r{  r  zdefault textr  r8  r9  r:  r   r   rW   )r;  r=  r>  r  r  r}  z"litellm.aadapter_completion(model=r?  rF  rG  r^   )r   r   r   r|  rI  rJ  r   r  r   r   r>  rk   rK  r   rr  r  r  r   r   r'  r   r   )r   r;  r   r0  rh  r#  rR  rQ  rU  r4  r  r)  s               rf   r=  Router._aadapter_completion  s}    B	!''8z&R  AH#BB#)nEF$*JJ/Dd$K  C   J
 //:/U./446DgJ>>% ? L Z(A-(22 ",#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&4ZL@WX OmF )
  . )=== *  	!&&4UG;OPSTUPVxW^_  &!+&G	r:  c                     XS'   X$S'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6nU$ ! [         a  nUeS nAff = f)Nr   r  r.  r|   r   r  r^   )
_embeddingrL  r|   r   r   r3  r'  r   r   r  r/  r0  r4  r)  s          rf   	embeddingRouter.embedding  s    		#7O#7O*.//F&'$*JJ}d>N>N$OF=!j"-44mU5KL33=f=HO 	G	r  c           	         S n [         R                  " SU SU 35        U R                  UUUR                  SS 5      S9nU R	                  XSS9  US   R                  5       nUS   nU R                  XSSS	9nUR                  S
S 5      nUb  Ub  XR                  :w  a  S n	OUn	U R                  U==   S-  ss'   U R                  US9  [        R                  " S0 0 UEUU R                  U	S.EUED6n
U R                  U==   S-  ss'   [         R                  " SU S35        U
$ ! [          aG  n[         R                  " SU S[#        U5       S35        Ub  U R$                  U==   S-  ss'   UeS nAff = f)NzInside embedding()- model: r{  r8  r   r  r8  r:  r   r   syncr  r%  rW   r;  r  r=  r>  zlitellm.embedding(model=r?  rF  rG  r^   )r   r   rH  rI  rJ  r   rK  rL  r%  r   rN  r   rE  rk   r   r   r'  r   r   )r   r  r   r0  rQ  r#  rR  rS  rT  rU  r4  r)  s               rf   rC  Router._embedding
  s   
4	!''-eWJvhG 66$*JJ/Dd$K 7 J
 //:/U./446DgJ%)%5%5%& &6 &" %jjD9O+*6#'E'EE#5Z(A-( 11Z1H(( "#33*	
 H z*a/*!&&*:,6MN O 	!&&*:,6J3q6(RYZ %
+q0+G	s   D"D' '
E81AE33E8c           
      "  #     XS'   X$S'   U R                   US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R
                  " [        U U[        R                  " 5       US95        UeS nAff = f7fr"  )	_aembeddingr2  rq  r'  rr  rs  r0   rv  rw  rD  s          rf   
aembeddingRouter.aembeddingB  s     	#7O#7O*.*:*:F&'00u0L!??I&IIHO J 		(,0#)(1(<(<(>'(	 G		r  c           	      $  #    S n [         R                  " SU SU 35        [        U5      nU R                  UUUR	                  SS 5      S9I S h  vN nU R                  XcS9  US   R                  5       nUS   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEUU R                  US	.EUED6n	U R                  UUS
S9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XeS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XeS9I S h  vN   U	I S h  vN n	U R"                  U==   S-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S-  ss'   UeS nAff = f7f)NzInside _aembedding()- model: r{  r8  rH  r:  r   r   rW   rJ  r  r  r}  zlitellm.aembedding(model=r?  rF  rG  r^   )r   r   r   r|  rI  rJ  r   r  r   r   rN  rk   rK  r   rr  r  r  r   r   r'  r   r   )r   r  r   r0  rQ  rh  r#  rR  rU  r4  r  r)  s               rf   rM  Router._aembedding[  sx    
A	!''/wjI  AH#BB$*JJ/Dd$K  C   J
 //:/U./446DgJ>>% ? L
 Z(A-()) "#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&+J<7NO OkD )
  . )=== *  	!&&+J<7KCPQF8SZ[ %
+q0+G	s   HAF< FB5F< FF< FF 	F)F*F.F< 9F:F< F8	F< F:6F< HF< F< FFF< F5$F'%F51F< :F< <
HAHHHc           
        #     XS'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7fNr   r.  r|   r   r  ri  r^   )_acreate_filerL  r|   r   r   rq  r'  rr  rs  r0   rv  rw  r$  s        rf   acreate_fileRouter.acreate_file  s     
	#7O*.*<*<F&'$*JJ}d>N>N$OF=!j"-44mU5KL!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   B>A*A7 .A5/A7 4B>5A7 7
B;5B66B;;B>c                 n  #     [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      S9I S h  vN nU R                  XBS9  US	   R                  5       nUS
   nU R                  UUS9nU R                  U==   S-  ss'   [        US
   S9u  p  n
[        US   US9US'   [        R                  " S0 0 UEU	U R                  US.EUED6nU R                  UUSS9nUb]  [        U[         R"                  5      (       a>  U IS h  vN    U R%                  XCS9I S h  vN   UI S h  vN nS S S 5      IS h  vN   O!U R%                  XCS9I S h  vN   UI S h  vN nU R&                  U==   S-  ss'   [         R(                  " SU S35        U$  GNb N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [*         aJ  n[         R,                  " SU SU S[/        U5       S35        Ub  U R0                  U==   S-  ss'   UeS nAff = f7f)Nr8  r{  r  files-api-fake-textr  r8  r9  r:  r   r   rW   r   r  )file_contentnew_model_namecustom_llm_providerr=  r>  r  r  r}  litellm.acreate_file(model=r?  , rF  rG  r^   )r   r   r   r|  rI  rJ  r   r  r   rS   r#   r   rU  rk   rK  r   rr  r  r  r   r   r'  	exceptionr   r   )r   r   r0  rh  r#  rR  rQ  rU  stripped_modelr^  r  r4  r  r)  s                 rf   rT  Router._acreate_file  s    
J	!''5eWJvhO  AH#BB#)6KLM$*JJ/Dd$K  C   J
 //:/U./446DgJ>>% ? L Z(A-( 9I7m95NA 4#F^NF6N ++ +>#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&-j\9PQ O}V )
  . )=== *  	!++-eWBvh>RSVWXSYRZZab  &!+&G	s   H5AG F5CG )F8*G -G F:	G F<G G F>G 3G4	G =G>6G 4H55G 8G :G <G >G  GG	GG G 
H2(AH--H22H5c           
        #     XS'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7frS  )_acreate_batchrL  r|   r   r   rq  r'  rr  rs  r0   rv  rw  r$  s        rf   acreate_batchRouter.acreate_batch
	  s     
	#7O*.*=*=F&'$*JJ}d>N>N$OF=!j"-44mU5KL!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		rW  c                 >  #     [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      S9I S h  vN n[        SS	9nUR                  U0 5      R                  US
   S   UR                  S0 5      UR                  S
0 5      R                  S5      S.5        UR                  S0 5      US'   US
   R                  5       nUS   nU R                  XBS9  U R                  UUS9nU R                  U==   S-  ss'   [        US   S9u  p  n	[        R                  " S0 0 UEU
U R                   US.EUED6nU R#                  UUSS9nUb]  [%        U[&        R(                  5      (       a>  U IS h  vN    U R+                  XCS9I S h  vN   UI S h  vN nS S S 5      IS h  vN   O!U R+                  XCS9I S h  vN   UI S h  vN nU R,                  U==   S-  ss'   [         R.                  " SU S35        U$  GN N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [0         aJ  n[         R2                  " SU SU S[5        U5       S35        Ub  U R6                  U==   S-  ss'   UeS nAff = f7f)Nz Inside _acreate_batch()- model: r{  r  rY  r  r8  r9  re  )function_namer   r   r  r  r  r:  rW   rZ  r]  r  r  r}  r_  r?  zlitellm._acreate_batch(model=r`  rF  rG  r^   )r   r   r   r|  rI  r"   r   r   rL  r   rJ  r  r   rS   r   rf  rk   rK  r   rr  r  r  r   r   r'  ra  r   r   )r   r   r0  rh  r#  metadata_variable_namerR  rQ  rU  r  r^  r4  r  r)  s                 rf   re  Router._acreate_batch"	  s    
P	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K  C   J
 &H.&" 4b9@@",-=">w"G",..r"B */? D H H T $.>>,#CF< ./446DgJ//:/U>>% ? L Z(A-( ,<$w-+P(AAq,, +>#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&-j\9PQ OIb )
  . )=== *  	!++/wb@TUXYZU[T\\cd  &!+&G	s   JAI HD=I H I H()H"*	H(3H$4H(8I H&I I	I %I&6I JI  I "H($H(&I (H?.H1/H?;I I 
JAJJJc           
        ^^#     U R                  5       nUc  [        S5      e/ mUU4S jn[        R                  " U Vs/ s H
  oC" U5      PM     snSS06I Sh  vN nU H  n[	        U[
        5      (       d  M  Us  $    T(       a  TS   e[        SR                  T5      5      es  snf  NQ! [         a:  n[        R                  " [        U T[        R                  " 5       US95        UeSnAff = f7f)	zh
Iterate through all models in a model group to check for batch

Future Improvement - cache the result.
NRouter not yet initialized.c                   >#     [        U S   S   S9u  p  n[        R                  " T5      nUR                  SS 5        [        R
                  " SSU0UD6I S h  vN $  N! [         a  nTR                  U5         S nAg S nAff = f7f)Nr   r   rZ  r^  r^   )rS   r   r   rI  r   aretrieve_batchr'  r   )rQ  r  r^  
new_kwargsr)  r0  receieved_exceptionss        rf   try_retrieve_batch2Router.aretrieve_batch.<locals>.try_retrieve_batch	  s       4D()9:7C40AAq "&v!6JNN#8$?!(!8!8 ",?"CM"    !  (//2 sA   B
AA" A A" B
 A" "
B,B=B
BB
return_exceptionsTr   z7Unable to find batch in any model. Received errors - {}ri  )get_model_listr'  rr  r  r   r7   r   rs  r0   rv  rw  )	r   r0  filtered_model_listrr  r   resultsr  r)  rq  s	    `      @rf   ro  Router.aretrieve_batchy	  s
    6	"&"5"5"7"* =>>#%  " $NN9LM9L$U+9LM"& G "fe,,!M "
 $*1-- IPP(  N(  		(,0#)(1(<(<(>'(	 G		sL   C7;B0 B)
B0 B.B0 =B0  C7.B0 0
C4:5C//C44C7c                   ^#    U R                  US9nUc  [        S5      eS[        4U4S jjn[        R                  " U Vs/ s H
  o" U5      PM     sn6 I Sh  vN nS/ SSSS.nU Hr  nUc  M  US	   c   [        US	5      (       a  [        US	5      US	'   [        US
5      US
'   US   R                  UR                  5        [        USS5      SL d  Mm  SUS'   Mt     U$ s  snf  N7f)zA
Return all the batches across all deployments of a model group.
rQ  Nrm  r   c                 |   >#     [         R                  " S0 0 U S   ETED6I S h  vN $  N! [         a     g f = f7f)Nr   r^   )r   alist_batchesr'  r/  s    rf   rr  0Router.alist_batches.<locals>.try_retrieve_batch	  sS     $22 ;/0;F;     s(   < , *, <, 
9<9<r   F)objectrR  first_idlast_idhas_morer  r  rR  r  T)	ru  r'  rA   rr  r  hasattrgetattrextendrR  )r   r   r0  rv  rr  rw  final_resultsr  s     `     rf   r|  Router.alist_batches	  s     #11U1C&9::	,? 	  5HI5HE '5HI
 

 
 F! ,49T9T07
0KM*-+269+Ei(f%,,V[[9 6:u5=04M*-  / J
s+   A C)C"C)C'C)2A!C)C)r.  c                    #    UR                  S5      (       a<  U R                  US   S9(       a%  U R                  US   S9I S h  vN nUS   S   US'   U" S0 UD6I S h  vN $  N N7f)Nr   rz  rZ  r   r^   )rL  ru  r|  )r   r.  r0  r#  s       rf   )_pass_through_moderation_endpoint_factory0Router._pass_through_moderation_endpoint_factory	  s}     
 ::g4#6#6&/#6#R#BBWo  C   J ))9:7CF7O&0000	 1s$   AA)A%A) A'!A)'A)r   )
assistantsr   c                 Z   ^ ^^   SS[         [        S      S[         S   4UUU 4S jjjnU$ )Nr^  r   azurer>  r   c                    >#    TS:X  a  TR                   " STU US.UD6I S h  vN $ TS:X  a  TR                  " SST0UD6I S h  vN $ g  N' N7f)Nr  )r.  r^  r>  r   r.  r^   ))_pass_through_assistants_endpoint_factoryr  )r^  r>  r0  r   r.  r   s      rf   new_function-Router.factory_function.<locals>.new_function	  s     
 L(!KK &7(;! 	   l*!KK &7   +s!    AA!AAAANN)r   r
   )r   r.  r   r  s   ``` rf   r   Router.factory_function	  s?     IM.2	!)'2C*D!E	]+	 	& re   r^  r  r>  c                    #    UcF  U R                   b.  U R                   S   nUR                  U R                   S   5        O[        S5      eU" SX#S.UD6I Sh  vN $  N7f)z@Internal helper function to pass through the assistants endpointNr^  r   z'custom_llm_provider' must be set. Either via:
 `Router(assistants_config={'custom_llm_provider': ..})` 
or
 `router.arun_thread(custom_llm_provider=..)`)r^  r>  r^   )rr   r   r'  )r   r.  r^  r>  r0  s        rf   r  0Router._pass_through_assistants_endpoint_factory
  s      &%%1&*&<&<=R&S#d445EFG s  ' 
 3
FL
 
 	
 
s   AA AA c                 	  #    UR                  S5      nUR                  SS5      nUR                  SU R                  5      nUR                  SU R                  5      nUR                  SU R                  5      n U R                  UUUUUS9  U R                  " U0 UD6I Sh  vN n[        R                  " S	U 35        U$  N! [         Ga  n	[        R                  " S
[        R                  " 5        35        U	n
SnUR                  S5      nSnUSL d  Uc  U	eU U
S.UEnSU;  a  U R                  US'   SU;  a  SUS'    [        R                  " S5        [        US9nU(       a/  UR                  UUS.5        [!        U0 UD6I Sh  vN  nUs Sn	A	$ [#        U	[$        R&                  5      (       a  UbD  U R)                  UUS9nUc  U
eUR                  UUS.5        [!        U0 UD6I Sh  vN  nUs Sn	A	$ SR+                  X6U5      n[        R                  " SR+                  U5      S9  U	=R,                  SR+                  U5      -  sl        O[#        U	[$        R.                  5      (       a  UbD  U R)                  UUS9nUc  U
eUR                  UUS.5        [!        U0 UD6I Sh  vN  nUs Sn	A	$ SR+                  X7U5      n[        R                  " SR+                  U5      S9  U	=R,                  SR+                  U5      -  sl        Ub  Ub  [        R                  " SU 35        [1        U[3        [4        U5      S9u  nnUc  Ub  UU   S   nUcJ  [        R                  " SU SU 35        [7        U
S5      (       a  U
=R,                  SU SU 3-  sl        U
eUR                  UUS.5        [!        U0 UD6I Sh  vN  nUs Sn	A	$ O! [         a  n[        R8                  " 5         [;        U5      n[        R<                  " S R+                  [5        U5      [        R                  " 5       [?        U US!9I Sh  vN  5      5        [5        U5      n SnAOSnAff = f[7        U
S5      (       aX  U
=R,                  S"R+                  UU5      -  sl        [A        U5      S:  a$  U
=R,                  S#R+                  U5      -  sl        U
eSn	A	ff = f7f)$zg
Try calling the function_with_retries
If it fails after num_retries, fall back to another model group
r   disable_fallbacksFr   r   r   )r0  r  r   r   r   NzAsync Response: 	TracebackrB  T)litellm_routerrm  r}   fallback_depthr   zTrying to fallback b/w models)r   )fallback_model_grouporiginal_model_group)r   r  zmodel={}. context_window_fallbacks={}. fallbacks={}.

Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContextWindowExceededError'. No context_window_fallback set. Defaulting                             to fallbacks, if available.{})msgz
{}zmodel={}. content_policy_fallback={}. fallbacks={}.

Set 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting                             to fallbacks, if available.{}zinside model fallbacks: r   z7No fallback model group found for original model_group=z. Fallbacks=rD  zlitellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}
{}

Debug Information:
Cooldown Deployments={}rj  rh  z;
Received Model Group={}
Available Model Group Fallbacks={}z
Error doing the fallback: {})!rL  rI  r   r   r   _handle_mock_testing_fallbacksr+  r   r   r'  rv  rw  r}   r   r+   r   r-   r   r   ContextWindowExceededError(_get_fallback_model_group_from_fallbacksr   rD  rP  r,   r   r   r  	print_excr   errorr(   r  )r   argsr0  r  r  r   r   r   r4  r)  rm  r  r  fallback_failure_exception_strinput_kwargsis_non_standard_fallback_formaterror_messagegeneric_fallback_idxnew_exceptionrh  s                       rf   rq  $Router.async_function_with_fallbacks)
  s    
 &,ZZ%8,2JJ7JE,R$*JJ{DNN$K	39::&(E(E4
  4:::&(E(E4
 	%//'#)A)A 0  "==tNvNNH!''*:8*(EFO O  s	%!'')I4H4H4J3K(LM!"#' 28**W2E -/* D(,@,H #'&8 L l2040B0B_-|312-.OD%**+JK 3V'3/ 3 ''4=4H &8&&&    H
 $Oa!C!CDD/; II*B,7 J  - 07"44$++8L8L *<!*** $ $  ( )q  )x  )x'9) .22!;;A6 -< 		V]]=%AA	7#F#FGG/; II*B,7 J  - 07"44$++8L8L *<!*** $ $  ( )p  )w  )w'9) .22!;;A6 -< 		V]]=%AA	([-D)//2J9+0VW0&/(,S+(> ?(*> -40</89M/Ns/S,+3-22UVaUbbnoxnyz ##5yAA.66<st  tA  AM  NW  MX  ;Y  Y600 ''4H4H &8&&&    H
 $O D##%#DV#L %++ o  v  vM*!,,.M48-=  	 25]1C.D )955"**.m.t.t(/ * 56:&..8??:. %$gs	%s   A8S';(C #C$C S'C S$A4SAOFOS$S'AO3G64O:S$;S' B3O3J64O:S$;S' DOOOS$S'S
Q1(A&Q,QQ,'S,Q11A.SS$$S'r  c                 J   UR                  SS5      nUR                  SS5      nUR                  SS5      nUb!  USL a  [        R                  " USSU SU 3S	9eUb!  USL a  [        R                  " USSU S
U 3S	9eUb"  USL a  [        R                  " USSU SU 3S	9egg)a  
Helper function to raise a litellm Error for mock testing purposes.

Raises:
    litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params
    litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params
    litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params
mock_testing_fallbacksNmock_testing_context_fallbacks%mock_testing_content_policy_fallbacksTrB  #This is a mock exception for model=z#, to trigger a fallback. Fallbacks=r   rE  rD  zF, to trigger a fallback.                     Context_Window_Fallbacks=zF, to trigger a fallback.                     Context_Policy_Fallbacks=)rI  r   InternalServerErrorr  rP  )	r   r0  r  r   r   r   r  r  r  s	            rf   r  %Router._handle_mock_testing_fallbacks
  s
     "(,Dd!K)/,d*
& 17

3T1
- "-2HD2P--!=k]Jmnwmxy  +6.$644!=k] K..F-GI  2=5=55!=k] K..F-GI  > >re   c                 >  #    [         R                  " SU SU 35        UR                  S5      nUR                  SU R                  5      n[	        U5      nUR                  SU R
                  5      nUR                  SU R                  5      nUR                  S5      nUR                  S5      n	UR                  S	5      =(       d    0 n
S
U
;   aI  [        U
S
   [        5      (       a1  U R                  U
S
   S9nUb  U
R                  S[        U5      05        [         R                  " SU SU	 35         U R                  XS9  U R                  " U/UQ70 UD6I S h  vN nU$  N! [         Ga'  nS nUn[!        USS 5      nUb  [        U["        5      (       a  Un	 U R%                  UR                  S5      =(       d    SUS9I S h  vN  u  nnU R'                  UUUUUUS9  U R(                  c  U R*                  b#  U R-                  XR                  S5      S9nUb  Un	U	S:  a  U R/                  X/S9nOe U R1                  UU	U	UUS9n[2        R4                  " U5      I S h  vN    [7        U	5       H  n U R                  " U/UQ70 UD6I S h  vN  n[8        R:                  " U5      (       a  UI S h  vN  nUs  s S nA$ ! [         a  nU R/                  X-S9nX-
  nUR                  S5      nUb  U R%                  UUS9I S h  vN  u  nnO/ nU R1                  UUU	UUS9n[2        R4                  " U5      I S h  vN     S nAM  S nAff = f   [=        U5      [>        R@                  ;   a  [C        USU	5        [C        USU5        UeS nAff = f7f)Nz+Inside async function with retries: args - z; kwargs - r.  r   r   r   r   r|   r   r  rz  model_group_sizez/async function w/ retries: original_function - z, num_retries - )r  r0  rB  r  )r  r   all_deploymentsr   regular_fallbacksr   )ra  r  r   )r0  r)  )r)  remaining_retriesr|   r   r  r   )"r   r   rI  r   r   r   r   rL  r   r   ru  r   r  %_handle_mock_testing_rate_limit_error	make_callr'  r  r   r  should_retry_this_errorr   r   r.   	log_retry_time_to_sleep_before_retryrr  r  r  inspectiscoroutinefunctionr   r   LITELLM_EXCEPTION_TYPESr  )r   r  r0  r.  r   rh  r   r   r  r|   	_metadatarq   r4  r)  current_attemptrm  deployment_num_retriesr  _all_deployments_retry_policy_retriesr   r  _modelr  _timeouts                            rf   r+  "Router.async_function_with_retries,  s    ##9${6(S	
 #JJ':;JJ{DNN;	<VD#)::&(E(E$
  $*::&(E(E$
  &,ZZ%8jj/ !**Z06B	I%*Y}5Ms*S*S,,	-8P,QJ%  "4c*o!FG##=>O=PP`al`mn	
c	%66' 7  "^^,=OOOOHO P  [	%"O!"%,Qt%D"%1j&7 7 5 99 **W-3%5 :    3 "2 (($8 0)A"+)A )  !!-00< )-(N(N0jj>Q )O )% )4"7KQvL ::$"-'$8 0 ; K --,,,#(#52%)^^4E%W%WPV%WWWH22   *2>>#O  2!^^6^?F(3(E%,2JJw,?F)"&"E"E&,1A #F #   0,a 02,#??,*;$/,@(8  @  H "--111+2 $6B &'7+J+JJ*M;G*M?K$$w[	%s   D4N7(E( E& E( %N&E( (N3AN
GBN'I*(N<K
J%K
;J><K
NNN

M:ML
7MM	
MNM>NNNc                    #    UR                  S5      nU" U0 UD6n[        R                  " U5      (       d  [        R                  " U5      (       a
  UI Sh  vN nU R	                  XTS9I Sh  vN   U$  N N7f)zN
Handler for making a call to the .completion()/.embeddings()/etc. functions.
r   N)r4  r  )rL  r  r  isawaitableset_response_headers)r   r.  r  r0  r  r4  s         rf   r  Router.make_call  sp      jj)$d5f5&&x00G4G4G4Q4Q%~H'''SSS	 &Ss$   AA9A5A9.A7/A97A9c                     UR                  SS5      nUb9  USL a3  [        R                  " SU 35        [        R                  " USSU S3S9egg)	z
Helper function to raise a mock litellm.RateLimitError error for testing purposes.

Raises:
    litellm.RateLimitError error when `mock_testing_rate_limit_error=True` passed in request params
mock_testing_rate_limit_errorNTzTlitellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=rB  r  z , to trigger a rate limit error.r  )rI  r   r   r   RateLimitError)r   r0  r  r  s       rf   r  ,Router._handle_mock_testing_rate_limit_error  st     9?

+T9
% *5-5!&&fgrfst ((!=k]Jjk  6 6re   r  r   r  r  c                 <   SnUb   [        U[        5      (       a  [        U5      nSnUb   [        U[        5      (       a  [        U5      n[        U[        R                  5      (       a  Ub  Ue[        U[        R
                  5      (       a  Ub  Ue[        U[        R                  5      (       a  Ue[        U[        R                  5      (       a  US::  a  Ub  [        U5      S:  a  Ue[        U[        R                  5      (       a	   US::  a  UeUS::  a  Ueg)aE  
1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None

2. raise an exception for RateLimitError if
    - there are no fallbacks
    - there are no healthy deployments in the same model group
r   rW   T)
r   r   r  r   r  rP  NotFoundErrorr   r  AuthenticationError)	r   r  r   r  r   r   r  _num_healthy_deployments_num_all_deploymentss	            rf   r  Router.should_retry_this_error  s   " $% *z:Mt/T/T'*+>'?$ &:ot+L+L#&#7  ug@@AA(4K ugAABB(4KeW2233KeV2233(A-%1)*Q.eV7788
 %) $q(Kre   c                   ^ ^^ SSK Jn  UUU 4S jn [        R                  " 5       nU" SS9 nUR	                  U5      nUR                  5       sSSS5        $ ! , (       d  f       g= f! [         a
    U" 5       s $ f = f)zf
Sync wrapper for async_function_with_fallbacks

Wrapped to reduce code duplication and prevent bugs.
r   )ThreadPoolExecutorc                  @  > [         R                  " 5       n  [         R                  " U 5        U R                  TR                  " T0 TD65      U R                  5         [         R                  " S5        $ ! U R                  5         [         R                  " S5        f = f)z9Run the coroutine in a new event loop within this thread.N)rr  new_event_loopset_event_looprun_until_completerq  close)new_loopr  r0  r   s    rf   run_in_new_loop7Router.function_with_fallbacks.<locals>.run_in_new_loop  s{    --/H-&&x02266GG  &&t,  &&t,s   6A5 5(BrW   )max_workersN)concurrent.futuresr  rr  get_running_loopsubmitr  RuntimeError)r   r  r0  r  r  r  executorfutures   ```     rf   r3  Router.function_with_fallbacks  sh     	:
	-	%((*A $2h!9}} 322  	%"$$	%s.   A* !A	A* 
A'#A* 'A* *A>=A>c                 r    Uc  gSnU H*  n[        UR                  5       5      S   U:X  d  M%  XB   n  U$    U$ )a  
Returns the list of fallback models to use for a given model group

If no fallback model group is found, returns None

Example:
    fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}]
    model_group = "gpt-3.5-turbo"
    returns: ["gpt-4"]
Nr   )r   keys)r   r   r  r  r  s        rf   r  /Router._get_fallback_model_group_from_fallbacks3  sO     48DDIIK #{2'+'8$##	  $#re   r)  r  c                    Ub  [        U5      S:X  a  O(Ub%  [        U[        5      (       a  [        U5      S:  a  gSn[        US5      (       a1  [        UR                  S5      (       a  UR                  R
                  n[        US5      (       a  UR                  nUb#  [        R                  " UUUU R                  S9nU$ [        R                  " UUU R                  S9nU$ )	z
Calculate back-off, then retry

It should instantly retry only when:
    1. there are healthy deployments in the same model group
    2. there are fallbacks for the completion call
NrW   r   r4  headerslitellm_response_headers)r  r   response_headersmin_timeout)r  r   r  )
r  r   r   r  r4  r  r  r   _calculate_retry_afterr   )r   r)  r  r|   r   r  r  r~   s           rf   r  "Router._time_to_sleep_before_retryL  s    " &3+?1+D+.55'(1,481j!!gajj)&D&D zz111011 99'44"3'!1 ,,	G  44"3' ,,G re   c                   #     UR                  SS5      nUc  [        S5      eUS   R                  S5      c  gUS   S   R                  SS5      nUR                  SS5      nUR                  SS5      nUb  Uc  g[        U[        5      (       a  [	        U5      n[        U5      n	UR                  S	S
5      n
[        5       nUR                  S5      n[        R                  R                  R                  XUS9nU R                  R                  UU
U	[        R                  R                  S9I Sh  vN   [        R                   R                  R                  XUS9nU R                  R                  USU	[        R                  R                  S9I Sh  vN   [#        U US9  U$  Nt N! [$         a8  n[&        R(                  " SR                  [	        U5      5      5         SnAgSnAff = f7f)z7
Track remaining tpm/rpm quota for model in model_list
standard_logging_objectNzstandard_logging_object is Noner   r   r#  r  model_idtotal_tokensr   %H-%Mr  current_minuter   keyr  rh  rc   rW   rj  deployment_idzOlitellm.router.Router::deployment_callback_on_success(): Exception occured - {})rL  r
  r   r   r   r   rU   strftimerG   TPMr  r   r   async_increment_cacher[   rc   RPMr3   r'  r   ra  )r   r0  completion_responserf  rg  r  deployment_namer  r  rh  r  dtr  tpm_keyrpm_keyr)  s                   rf   r   %Router.deployment_callback_on_success  s    F	HN

)4I# '. !BCC&'++J7?"()9"::"F"J"J $# 699-N,00TB&"*C((RB#DV#L &=&A&A.RS&T
 &'!#" *--33:: ;  jj66&%5#--	 7    *--33:: ;  jj66%5#--	 7    B,0"$
 /  	!++ahhF
 	sk   G25F- G2AF- <G2=B7F- 4F)5A#F- F+F- (G2)F- +F- -
G/7.G*%G2*G//G2c                 0   SnUS   R                  S5      c  OoUS   S   R                  SS5      nUS   R                  S0 5      =(       d    0 nUR                  SS5      nUb  Uc  g[        U[        5      (       a  [        U5      nUb  [	        U US9nU$ g)z
Tracks the number of successes for a deployment in the current minute (using in-memory cache)

Returns:
- key: str - The key used to increment the cache
- None: if no key is found
Nr   r   r  r  r  r  )rL  r   r   r   r3   )	r   r0  r  rf  rg  r  r  r  r  s	            rf   r   *Router.sync_deployment_callback_on_success  s     "#''
3; !12:>BB=RVWK 0155lBGM2Jd+B"bjB$$W>C(, C Jre   c                 @    UR                  SS5      n[        USS5      nUR                  S0 5      R                  S0 5      n[        R                  R                  R                  US9nUR                  S0 5      R                  SU R                  5      n	Ub2  [        R                  R                  US	9n	U	b  U	S
:  a  U R                  n	[        U[        5      (       a+  UR                  SS5      n
[        U U
S9  [        U UUU
U	S9nU$ g! [         a  nUeSnAff = f)a(  
2 jobs:
- Tracks the number of failures for a deployment in the current minute (using in-memory cache)
- Puts the deployment in cooldown if it exceeds the allowed fails / minute

Returns:
- True if the deployment should be put in cooldown
- False if the deployment should not be put in cooldown
ra  Nstatus_coderB  r   r  )rm  r   )r  r   r  r  rj  exception_statusrm  r#  time_to_cooldownF)rL  r  r   litellm_core_utilsexception_mapping_utils_get_response_headersr   utils&_get_retry_after_from_exception_headerr   r   r2   r*   r'  )r   r0  r  rf  rg  ra  r  _model_infoexception_headers_time_to_cooldownr  r  r)  s                rf   r   %Router.deployment_callback_on_failure  s?    ,	

;5I&y-D **%5r:>>|RPK ' : : R R h h#, !i ! !'

+;R @ D D!3!3! !, MMHH): I  " %,0AA0E(,(:(:%+t,, +d ;@,0"/ 3,0%5'0,%6  	G	s   DD 
DDDr  c                 2  #    US   S   R                  SS5      nUS   S   R                  SS5      nUS   R                  S0 5      =(       d    0 nUR                  SS5      nUb  Uc  g[        U[        5      (       a  [        U5      n[	        U5      n	[        5       n
U
R                  S5      n[        R                  R                  R                  XUS	9nU R                  R                  US
U	[        R                  R                  S9I Sh  vN   g N7f)z#
Update RPM usage for a deployment
r   r   r#  Nr  r  r  r  r  rW   r  )rL  r   r   r   r   rU   r  rG   r  r  r   r   r   r[   rc   )r   r0  r  rf  rg  r  r  r  r  rh  r  r  r  s                rf   r   +Router.async_deployment_callback_on_failure2  s     !!12:>BB$
 -.z:>>}dS,-11,CIr
^^D$'"*C  RB<VD

 "%%++22 3 
 jj..-%%	 / 
 	
 	
s   DDDDc                     [        U5      R                  [        U5      S.nUR                  5        Ha  u  nnUS;  a  XSU'   M  US:X  d  M  [	        U[
        5      (       d  M1  0 US'   US   R                  5        H  u  pgUS:w  d  M  XsU   U'   M     Mc     [        U R                  5      S:  a  U R                  R                  S5        U R                  R                  U5        U R                  US   S'   U$ ! [         a  nUeSnAff = f)zs
When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
)exception_typeexception_string)r   r,  r.  r   r      r   N)r   r_   r   r  r   r   r  r   rI  r   r'  )r   r0  r)  previous_modelr  r  
metadata_k
metadata_vs           rf   r  Router.log_retryT  s   	 #'q'"2"2$'FN 	 II()1%*_At)<)<13N:.282D2J2J2L.
%)::<F1-j9 3M  4''(1,$$((+  ''7484H4HF:01M 	G	s+   AC; C; $!C; 	A1C; ;
DDDr  rh  c                     UnU R                   R                  X2SS9nUc  SnU R                   R                  X4SSS9  U$ US-  nU R                   R                  X4SS9  U$ )zF
Update deployment rpm for that minute

Returns:
- int: request count
T)r  rh  
local_onlyrW   r]   )r  r  r#  rc   )r  r  r#  )r   	get_cache	set_cache)r   r  rh  r  request_counts        rf   _update_usageRouter._update_usagev  s      

,,t - 
  MJJ  Tr !   QMJJ  T !  re   r  r  exception_strc                 (   U R                  US9nUb  [        U5      S:X  a  g S/nUb  U H
  nXc;   d  M
    g   [        U[        5      (       a  [	        U5      nUS:  a#  US:  a  US:X  a  gUS	:X  a  gUS
:X  a  gUS:X  a  ggg! [
         a     gf = f)a"  
A function to determine if a cooldown is required based on the exception status.

Parameters:
    model_id (str) The id of the model in the model list
    exception_status (Union[str, int]): The status of the exception.

Returns:
    bool: True if a cooldown is required, False otherwise.
r  rW   FAPIConnectionErrori  i  i  Ti  i  i  )get_model_groupr  r   r   r   r'  )r   r  r  r)  r  ignored_stringsignored_strings          rf   _is_cooldown_requiredRouter._is_cooldown_required  s    " **h*7"s;'71'<%	34O)&5N%6$ '6 *C00#&'7#8 3&+;c+A#s*%,%,%, !  		s-   B B 4B -B 4B ;B 
BBc                     U R                   c  gU R                    H"  n[        U[        5      (       d  M  SU;   d  M"    g   g)NFr   T)r   r   r   )r   fallbacks     rf   _has_default_fallbacksRouter._has_default_fallbacks  s9    >>!H(D))(? ' re   r4  c                 `   UR                   S   R                  S:w  a  gUR                  SU R                  5      nUb6  SnU H)  n[	        UR                  5       5      S   U:X  d  M%  Xa   n  O   Ub  gOU R                  5       (       a  g[        R                  " SR                  X5      5        g)z
Determines if a content policy error should be raised.

Only raised if a fallback is available.

Else, original response is returned.
r   content_filterFr   NTzyContent Policy Error occurred. No available fallbacks. Returning original response. model={}, content_policy_fallbacks={})
choicesfinish_reasonrL  r   r   r  r4  r   r   r   )r   r   r4  r0  r   r  r  s          rf   rO  )Router._should_raise_content_policy_error  s     A,,0@@#)::&(E(E$
 
 $/#' 0		$Q'50+/;( 1
 $/ 0((**"" H  O  O	

 re   c                     / n U R                  US9u  pC[        U[        5      (       a  / $  [	        XS9n/ nU H"  nUS   S   U;   a  M  UR                  U5        M$     Xc4$ ! [         a     NBf = f)NrZ  r  r  r  )#_common_checks_available_deploymentr   r   r'  r)   r   r   r   rh  r  r  unhealthy_deploymentsr   r#  s           rf   _get_healthy_deploymentsRouter._get_healthy_deployments  s    !#	"&"J"J #K #A *D11	 2
 !:$(!
 %'*J,'-1FF#**:6	 + #44  		s   'A# #
A0/A0c                   #    / n U R                  US9u  pC[        U[        5      (       a  / U4$  [	        XS9I Sh  vN n/ nU H"  nUS   S   U;   a  M  UR                  U5        M$     Xc4$ ! [         a     NJf = f NA7f)z
Returns Tuple of:
- Tuple[List[Dict], List[Dict]]:
    1. healthy_deployments: list of healthy deployments
    2. all_deployments: list of all deployments
rZ  r  Nr  r  )r<  r   r   r'  r'   r   r=  s           rf   r  %Router._async_get_healthy_deployments  s      "$	"&"J"J #K #A *D11+++ 2
 'F$('
 !
 %'*J,'-1FF#**:6	 +
 #44  		!
s1   B)A/ BA?1B/
A<9B;A<<Bc                     [         R                   H+  n[        U[        5      (       d  M  UR	                  U5        M-     g)z
Mimics 'async_routing_strategy_pre_call_checks'

Ensures consistent update rpm implementation for 'usage-based-routing-v2'

Returns:
- None

Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
N)r   r  r   r   r  )r   r#  r  s      rf   rN  'Router.routing_strategy_pre_call_checks0  s/     !**I)\22((4 +re   r  c           
        #    [         R                   H4  n[        U[        5      (       d  M   UR	                  X5      I Sh  vN   M6     g N	! [         R
                   a  nUb  [        R                  " UR                  U[        R                  " 5       [        R                  " 5       S95        [        R                  " UR                  U[        R                  " 5       4S9R                  5         [!        U UR"                  UUS   S   U R$                  S9  UeSnAf[&         a  nUb  [        R                  " UR                  U[        R                  " 5       [        R                  " 5       S95        [        R                  " UR                  U[        R                  " 5       4S9R                  5         UeSnAff = f7f)  
For usage-based-routing-v2, enables running rpm checks before the call is made, inside the semaphore.

-> makes the calls concurrency-safe, when rpm limits are set for a deployment

Returns:
- None

Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
Nra  traceback_exceptionrg  targetr  r  r  r  )r   r  r   r   async_pre_call_checkr  rr  rs  async_failure_handlerrv  rw  ro  	threadingThreadfailure_handlerstartr*   r  r   r'  )r   r#  rh  r  r  r)  s         rf   r  -Router.async_routing_strategy_pre_call_checks@  s|    " !**I)\22(#88VVV + W-- ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'-04)*+,#-l#;D#A)-);); G  ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'GsH   'GAAAGAF?"B9DF?(BF::F??Grk  c           
        #    Un[         R                   H6  n[        U[        5      (       d  M   UR	                  UUUUUS9I Sh  vN nM8     U$  N
! [
         a  n	Ub  [        R                  " UR                  U	[        R                  " 5       [        R                  " 5       S95        [        R                  " UR                  U	[        R                  " 5       4S9R                  5         U	eSn	A	ff = f7f)rF  r   r   r,  rk  rh  NrG  rI  )r   r  r   r   async_filter_deploymentsr'  rr  rs  rL  rv  rw  ro  rM  rN  rO  rP  )
r   r   r   r,  rh  rk  r  returned_healthy_deploymentsr  r)  s
             rf   !async_callback_filter_deployments(Router.async_callback_filter_deployments}  s     ( (;$ **I)\22'@@"'0L%-+9-= A   1 +8 ,+1 ! ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'Gs;   )C7AAAC7A
C4BC//C44C7r   c                     UnUR                  5        H  u  pE[        U[        5      (       a  X4-  nO=[        U[        5      (       a  U[        R
                  " U5      -  nOU[        U5      -  n[        U[        5      (       a  X5-  nMw  [        U[        5      (       a  U[        R
                  " U5      -  nM  U[        U5      -  nM     [        R                  " UR                  5       5      nUR                  5       $ )z
Helper function to consistently generate the same id for a deployment

- create a string from all the litellm params
- hash
- use hash as id
)
r  r   r   r   jsondumpshashlibsha256encode	hexdigest)r   r  r   
concat_strr  r  hash_objects          rf   _generate_model_idRouter._generate_model_id  s     !
"((*DA!S!!
At$$djjm+
c!f$
!S!!
At$$djjm+
c!f$
 + nnZ%6%6%89$$&&re   deployment_info_model_name_litellm_paramsr  c           
         [        S0 UDU[        S0 UD6US.D6nUR                  R                  nUR                  R                  b  UR                  R                  S-   U-   n[
        R                  " X$0S9  U R                  US9SLa4  [        R                  " SUR                   SUR                  S	    35        gU R                  US9nUR                  SS
9nU R                  R                  U5        U$ )a.  
Create a deployment object and add it to the model list

If the deployment is not active for the current environment, it is ignored

Returns:
- Deployment: The deployment object
- None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
)rQ  r   r  N/)
model_costr;  TzIgnoring deployment z% as it is not active for environment supported_environmentsr   r^   )r@   rC   r   r   r^  r   register_model$deployment_is_active_for_environmentr   warningrQ  r  _add_deploymentto_jsonrq   r   )r   rc  rd  re  r  r#  r   s          rf   _create_deploymentRouter._create_deployment  s&       

")<O<"	

 !//55$$88D))==CkQ  		
 44
4KSWW!))&z'<'<&==bcmcxcx  zR  dS  cT  U ))Z)@
"""5u%re   c           	      b   UR                   b   SUR                   ;  d  UR                   S   c  g[        SS9nUc  [        S5      eU[        ;  a  [        S[         SU 35      eUR                   S    H%  nU[        ;  d  M  [        S[         SU S	U 35      e   X!R                   S   ;   a  gg
)a  
Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments

Requires `LITELLM_ENVIRONMENT` to be set in .env. Valid values for environment:
    - development
    - staging
    - production

Raises:
- ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values
- ValueError: If supported_environments is not set in model_info or not one of the valid values
ri  TLITELLM_ENVIRONMENT)secret_namezPSet 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .envz#LITELLM_ENVIRONMENT must be one of z. but set as: z&supported_environments must be one of z for deployment: F)r  r   r
  r;   )r   r#  litellm_environment_envs       rf   rk  +Router.deployment_is_active_for_environment  s     !!)'z/D/DD$$%=>F,9NO&b  &@@56P5QQ_`s_tu  ))*BCD55 <=W<XXfgkfll}  I  ~J  K  D "7"78P"QQre   c           	         [         R                  " U5      n/ U l        U GH  nUR                  S5      nUR                  S5      n[	        U[
        5      (       aU  UR                  5        HA  u  pg[	        U[        5      (       d  M  UR                  S5      (       d  M4  [        U5      XV'   MC     UR                  S0 5      nSU;  a  U R                  XE5      n	XS'   UR                  SS 5      b=  [	        US   [        5      (       a%  US    H  n
XS'   U R                  UUUUS9  M     GM  U R                  UUUUS9  GM"     [        R                  " SU R!                  5        35        U Vs/ s H  oS   PM	     snU l        g s  snf )	NrQ  r   os.environ/r  r  organization)rc  rd  re  r  z
Initialized Model List )r   r   rq   rI  r   r   r  r   
startswithrT   ra  rL  r   ro  r   r   get_model_namesrj   )r   rq   original_model_listr   rd  re  r  r  r  _idorgr   s               rf   r   Router.set_model_list%  s   "mmJ7 )E))L1K#ii(89O/400+113DA!!S))all=.I.I-7]* 4 !&		, ;K ;&--kK$'D!"">48D/J J +>:C69N3++(-$/(7$/	 ,  ; ''$) +$3 +	 ( 9 )F 	##'(<(<(>'?@	
 6@@ZlOZ@@s   /Fc           	         SS K nU R                  R                  UR                  R                  5        UR                  R
                  c)  [        USS 5      b  [        US5      UR                  l        UR                  R                  c)  [        USS 5      b  [        US5      UR                  l        [        R                  " UR                  R                  UR                  R                  SS 5      S9u  nnnnSUR                  ;   a}  U R                  R                  UR                  UR                  SS95        UR                  R                   (       a/  U R"                  R                  UR                  R                   5        UR                  R                  S	/ 5      =(       d    / nU Hq  nUR                  S
0 5      n	S HV  n
X;   d  M
  X   R%                  S5      (       d  M$  X   R'                  SS5      nUR(                  R                  US5      X'   MX     Ms     U[        R*                  ;  a  [-        SU 35      e[.        R0                  " XR                  SS9S9  [        R2                  SL ax  [5        S5          SUR                  R                  ;   aO  UR                  R6                  c8  [        R8                  R;                  UR                  S S9nXR                  l        U$ U$ ! [,         aN  n[<        R>                  " SRA                  UR                  R                  [C        U5      5      5         S nAU$ S nAff = f)Nr   rpmtpmr^  r   r^  r   Tr   dataSources
parameters)endpointr  rx  rB  zUnsupported provider - rj  r   zAuto inferring regionr  )r   modez1Unable to get the region for azure model - {}, {})"osr   r   r   r   r  r  r  r   rS   rL  rQ  r   add_patternrn  r  r  r   rz  replaceenvironprovider_listr'  r$   
set_clientenable_preview_featuresprintregion_namer  get_model_regionr   r   r   r   )r   r#  r  r  r^  rT  r  data_sourcesdata_sourcer   	param_keyenv_nameregionr)  s                 rf   rm  Router._add_deploymentR  s    	$$Z%>%>%D%DE
 %%))1
E40<,3J,FJ%%) %%))1
E40<,3J,FJ%%) $$++11 * 9 9 = =%t!
	
$ *''' ++%%z'9'9t'9'L $$''44;;J<Q<Q<T<TU "0044]BGM2'K __\26F0	&6+<+G+G+V+V%088KH(*

x(DF%	 1 ( g&;&;;56I5JKLL 	!++$(0B0BPT0B0U	

 **d2)*z88>>>"11==E$]];;'1'@'@t < F =C--9 z  %++GNN"1177Q
 s   A)K5 5
M?AMMc                    UR                   R                  U R                  5       ;   a  gUR                  SS9nU R                  R                  U5        U R                  US9  U R                  R                  UR                  5        U$ )z
Parameters:
- deployment: Deployment - the deployment to be added to the Router

Returns:
- The added deployment
- OR None (if deployment already exists)
NTr   r;  )	r  r  rM  rn  rq   r   rm  rj   rQ  )r   r#  _deployments      rf   add_deploymentRouter.add_deployment  s       ##t'9'9';; !((d(;{+ 	
3 	
 5 56re   c                    UR                   R                  =(       d    SnU R                  US9nUb}  UR                  UR                  :X  a  gSn[	        U R
                  5       H)  u  pVUS   S   UR                   R                  :X  d  M'  UnM+     Ub  U R
                  R                  U5        U R                  US9  U$ )z
Add or update deployment
Parameters:
- deployment: Deployment - the deployment to be added to the Router

Returns:
- The added/updated deployment
rB  r  Nr  r  r;  )r  r  get_deploymentr   r  rq   rI  r  )r   r#  _deployment_model_id_deployment_on_routerremoval_idxr  r   s          rf   upsert_deploymentRouter.upsert_deployment  s      *4477=26:6I6I) 7J 7
 !,((,A,P,PP *.K'8
&t,
0E0E0H0HH"%K 9 &##K0 	z2re   r  c                     Sn[        U R                  5       H  u  p4US   S   U:X  d  M  UnM      Ub  U R                  R                  U5      nU$ g! [         a     gf = f)z
Parameters:
- id: str - the id of the deployment to be deleted

Returns:
- The deleted deployment
- OR None (if deleted deployment not found)
Nr  r  )r  rq   rI  r'  )r   r  deployment_idxr  r   r  s         rf   delete_deploymentRouter.delete_deployment  sm     0FCt$*!$ 1	)**>: 		s   A 
A A c                 "   U R                    H  nSU;   d  M  SUS   ;   d  M  XS   S   :X  d  M#  [        U[        5      (       a  [        S0 UD6s  $ [        U[        5      (       a  Us  $ [	        SR                  [        U5      5      5      e   g)T
Returns -> Deployment or None

Raise Exception -> if model found in invalid format
r  r  zModel invalid format - {}Nr^   rq   r   r   r@   r'  r   r   )r   r  r   s      rf   r  Router.get_deployment  s     __Eu$|1D)D\2488!%..)2E22#E:66$'(C(J(J4PU;(WXX % re   model_group_namec                     U R                    Hj  nUS   U:X  d  M  [        U[        5      (       a  [        S0 UD6s  $ [        U[        5      (       a  Us  $ [	        SR                  [        U5      5      5      e   g)r  rQ  zModel Name invalid - {}Nr^   r  )r   r  r   s      rf   "get_deployment_by_model_group_name)Router.get_deployment_by_model_group_name  sm     __E\"&66eT**%...z22 L#$=$D$DT%[$QRR % re   received_model_namec                     g r   r^   r   r#  r  r  s       rf   get_router_model_infoRouter.get_router_model_info+       	re   c                     g r   r^   r  s       rf   r  r  1  r  re   c                    Ub!  U R                  US9nUb  UR                  SS9nUc  [        S5      eUR                  S0 5      R                  SS5      nUc"  UR                  S0 5      R                  SS5      nUn[        R
                  " UR                  S0 5      R                  S	S
5      [        S0 UR                  S0 5      D6S9u  px  n	US:X  a  Uc  [        R                  " S5        OUS:w  a  UnU R                  R                  U5      n
SU;   at  U
bq  U
 Hk  n UR                  S0 5      R                  S5      UR                  S0 5      R                  S5      :X  a#  UR                  S0 5      R                  S	5      n  OMm     UR                  U5      (       d  SR                  X5      nOUn[        R                  " US9nUR                  S0 5      nUR                  U5        U$ ! [         a     M  f = f)a  
For a given model id, return the model info (max tokens, input cost, output cost, etc.).

Augment litellm info with additional params set in `model_info`.

For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.

Returns
- ModelInfo - If found -> typed dict with max tokens, input cost, etc.

Raises:
- ValueError -> If model is not mapped yet
Nr  Tr   zDeployment not foundr  
base_modelr   r   rB  r   r   r  zCould not identify azure model. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-modelsr   r  z{}/{}rZ  r^   )r  r   r
  rL  r   rS   rC   r   r  r   router'  rz  r   get_model_infor   )r   r#  r  r  r  r  r   r  r^  r  potential_modelspotential_modelmodel_info_namer  user_model_infos                  rf   r  r  7  s   & >--r-:K&(333F
344  ^^L"599,M
#(8"=AA,PTUJ -4,D,D..!126::7BG)QJNN;KR,PQ-
)Q ')j.@!'' O !G+E#22889LMe| 0 <'7O	*..|R@DD '^^L"=AA$GH %4$7$78H"$M$Q$Q '%E "H (8  344%nn-@HO#O++/B
 %..r:/*! % s    A%G..
G<;G<c                 n    U R                    H%  nSU;   d  M  SUS   ;   d  M  XS   S   :X  d  M#  Us  $    g)z
For a given model id, return the model info

Returns
- dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
- None: could not find deployment in list
r  r  Nrq   )r   r  r   s      rf   r  Router.get_model_info  s@     __Eu$|1D)D|,T22 L % re   c                 P    U R                  US9nUc  gUS   nU R                  US9$ )zD
Return list of all models in the same model group as that model id
r+  NrQ  rz  )r  ru  )r   r  r  rQ  s       rf   r-  Router.get_model_group  s>    
 ((B(/
-
""j"99re   user_facing_model_group_namec                 	   SnSnSnSnU R                  US9nUc  gU GH  nSn	SU;   a  US   U:X  a  Sn	O$SU;   a  U R                  R                  U5      b  Sn	U	(       d  ME  [        S0 US   D6n
U
R                  nSnUc  UR                  SS5      nUc"  UR                  S0 5      R                  SS5      nUc"  UR                  S0 5      R                  SS5      nSnUc  UR                  S	S5      nUc"  UR                  S0 5      R                  S	S5      nUc"  UR                  S0 5      R                  S	S5      n [        R                  " U
R                  S
9nSu  p [        R                  " U
R                  U
R                  S9u  p  nUc+  [        R$                  " XS9nUc  / n['        USSSSSUSUSS9
nUc  [)        SX//S.UD6nGOXR*                  ;  a  UR*                  R-                  U5        UR                  SS5      b0  US   b*  UR.                  b  US   UR.                  :  a
  US   Ul        UR                  SS5      b0  US   b*  UR0                  b  US   UR0                  :  a
  US   Ul        UR                  SS5      b*  UR2                  b  US   UR2                  :  a
  US   Ul        UR                  SS5      b*  UR4                  b  US   UR4                  :  a
  US   Ul        UR                  SS5       b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   b
  US   Ul        UR                  SS5      b  Uc  UR                  S5      nUR                  S	S5      b  Uc  UR                  S	5      nUb	  Uc  SnXK-  nUc  GM  Uc  SnX\-  nGM     Ub  Ub  XCl        Ub  XSl         Ub  UUl        U$ ! [         a    Sn GNf = f! [        R                  R                   a9  n[        R                  " SR!                  [#        U5      5      5         SnAGNSnAff = f)z
For a given model group name, return the combined model info

Returns:
- ModelGroupInfo if able to construct a model group
- None if error constructing model group info
Nrz  FrQ  Tr   r  r  r  rZ  )rB  rB  r  z.litellm.router.py::get_model_group_info() - {}r   r   )
r  
max_tokensmax_input_tokensmax_output_tokensinput_cost_per_tokenoutput_cost_per_tokenlitellm_providerr  supported_openai_paramssupports_system_messages)r  	providersr  r  r  r  "supports_parallel_function_callingsupports_visionsupports_function_callingr  r^   )!ru  r   r  rC   #configurable_clientside_auth_paramsrL  r   r  r   r'  rS   r^  
exceptionsBadRequestErrorr   r  r   r   get_supported_openai_paramsModelMapInforD   r  r   r  r  r  r  r  r  r  r  r  r  )r   r  r  model_group_info	total_tpm	total_rpmr  rq   r   is_matchr   _deployment_tpm_deployment_rpmr  litellm_modelrE  r  r)  r  s                      rf   _set_model_group_infoRouter._set_model_group_info  s^    6:#'	#'	SW+((K(@
EH%%*=*L%''--k:F+Fe4D.EFN BB 0 .2O&"'))E4"8&"')),<b"A"E"EeT"R&"'))L""="A"A%"N .2O&"'))E4"8&"')),<b"A"E"EeT"R&"'))L""="A"A%"N"$33.:N:NO
 +1'M4;4L4L(..(6(J(J51Q !*1*M*M'+' +2.0+)##%)&*)**+%1,C-1
  '#1 $ <$Zd$   'A'AA$..55lCNN#5t<H"#56B(99A%&89*;;< 9CCU8V$5NN#6=I"#67C(::B%&9:*<<= :DDW9X$6>>"8$?K$99A!"89&;;< =G.=$9 >>"94@L$::B!"9:&<<= >H/>$: NN#GN "#GHDPJN$GNN#4d;G"#45=7;$4NN#>EQ"#>?4GAE$>NN#<dCO"#<=I?I1@$< >>%.:?V&0nnU&;O>>%.:?V&0nnU&;O*$ !I,	*$ !I,	A  B '$'0$$'0$ 3>7 !D  M  "!
" %%55 %++DKKCPQFS s*    P#-P'P$#P$'Q>.Q99Q>c                     XR                   ;   a\  U R                   U   n[        U[        5      (       a  UnO%[        U[        5      (       a  US   SL a  gUS   nOgU R	                  UUS9$ U R	                  XS9$ )z
For a given model group name, return the combined model info

Returns:
- ModelGroupInfo if able to construct a model group
- None if error constructing model group info or hidden model group
hiddenTNr   )r  r  )r   r   r   r   r  )r   r  r  _router_model_groups       rf   get_model_group_infoRouter.get_model_group_infoa  s     000))+6D$$$&*#D$''>T)*.w-'--/-8 .   ))# * 
 	
re   c           	      *  #    [        5       nUR                  S5      n/ n/ nU R                  US9nUc  gU H  nUR                  S0 5      R                  S5      nUS   R                  S5      n	Ub  U	c  M@  UR	                  [
        R                  R                  R                  UU	US	95        UR	                  [
        R                  R                  R                  UU	US	95        M     XE-   n
U R                  R                  U
S
9I Sh  vN nUc  gUS[        U5       nU[        U5      S nSnUb)  U H#  n[        U[        5      (       d  M  Uc  SnX-  nM%     SnUb*  U H$  n[        U[        5      (       d  M  Uc  SnUU-  nM&     UU4$  N7f)z
Returns current tpm/rpm usage for model group

Parameters:
- model_group: str - the received model name from the user (can be a wildcard route).

Returns:
- usage: Tuple[tpm, rpm]
r  rz  Nr  r  r  r   r   )r  r   r  )r  r   )rU   r  ru  rL  r   rG   r  r  r   r  r   async_batch_get_cacher  r   r   )r   r  r  r  tpm_keysrpm_keysrq   r   r  r  combined_tpm_rpm_keyscombined_tpm_rpm_valuestpm_usage_listrpm_usage_list	tpm_usager  	rpm_usages                    rf   get_model_group_usageRouter.get_model_group_usage  s     
 ! ((K(@
E %		, ; ? ? EB+01A+B+F+F,M z]2OO##))00'#1 1  OO##))00'#1 1   * !) 3(,

(H(H& )I )
 #
 #*)@3x=)Q)@X)Q $(	%#a%% ($%	NI	 $ $(	%#a%% ($%	NI	 $
 )##3#
s   D	FFAF*F?Fc                 .  #    U R                  U5      I S h  vN u  p#U R                  U5      nUb  UR                  b  UR                  nOS nUb  UR                  b  UR                  nOS n0 nUb  Ub  XR-
  US'   XWS'   Ub  Ub  Xc-
  US'   XgS'   U$  N{7f)Nx-ratelimit-remaining-tokenszx-ratelimit-limit-tokensx-ratelimit-remaining-requestszx-ratelimit-limit-requests)r  r  r  r  )r   r  current_tpmcurrent_rpmr  	tpm_limit	rpm_limitreturned_dicts           rf   get_remaining_model_group_usage&Router.get_remaining_model_group_usage  s     )-)C)CK)P#P 44[A',<,@,@,L(,,II',<,@,@,L(,,II [%<<E<SM898A45 [%<>G>UM:;:C67- $Qs   BBA<Bc                   #    [        U[        5      (       a  [        US5      (       a  [        UR                  [        5      (       a  UR                  R                  S0 5        UUR                  S   S'   UR                  S   nSU;  aD  SU;  a>  Ub;  U R                  U5      I Sh  vN nUR                  5        H  u  pVUc  M
  XcU'   M     U$  N(7f)z
Add the most accurate rate limit headers for a given model response.

## TODO: add model group rate limit headers
# - if healthy_deployments > 1, return model group rate limit headers
# - else return the model's rate limit headers
r  r  zx-litellm-model-groupr  r  N)r   r   r  r  r   r   r  r  )r   r4  r  r  remaining_usageheaderr  s          rf   r  Router.set_response_headers  s      x++"233822D99##../CRH  ##$89' "*!8!89M!N /6HH4<NN+(,(L(L) # &5%:%:%<MF(5:62 &= #s   B&C(C)CCrQ  c                     / nU R                    HS  nSU;   d  M  SUS   ;   d  M  US   S   nUb  US   U:X  a  UR                  U5        M=  Ub  MB  UR                  U5        MU     U$ )zD
if 'model_name' is none, returns all.

Returns list of model id's.
r  r  rQ  )rq   r   )r   rQ  idsr   r  s        rf   rM  Router.get_model_ids  so     __Eu$|1D)D<(.)eL.AZ.OJJrN'JJrN % 
re   model_aliasc                     / nU R                    HT  nUc  M  US   U:X  d  M  Ub-  [        R                  " U5      nX%S'   UR                  U5        MC  UR                  U5        MV     U$ )zM
Return all deployments of a model name

Used for accurate 'get_model_list'.
rQ  )rq   r   r   r   )r   rQ  r  returned_modelsr   alias_models         rf   _get_all_deploymentsRouter._get_all_deployments  si     68__E%%*=*K*"&--"6K0;-#**;7#**51 % re   c                 n    U R                  5       nUc  / $ / nU H  nUR                  US   5        M     U$ )zV
Returns all possible model names for router.

Includes model_group_alias models too.
rQ  )ru  r   )r   rq   rj   r   s       rf   r{  Router.get_model_names+  sD     ((*
IAq/ re   c           
         [        U S5      (       Ga,  / nUb  UR                  U R                  US95        [        U S5      (       a  U R                  R	                  5        Hm  u  p4[        U[        5      (       a  UnO2[        U[        5      (       a  [        S
0 UD6nUS   SL a  MG  US   nOMO  UR                  U R                  XSS95        Mo     [        U5      S	:X  aI  U R                  R                  U5      nUb+  UR                  U Vs/ s H  n[        S
0 UD6PM     sn5        Uc  X R                  -  nU$ U$ gs  snf )z.
Includes router model_group_alias'es as well
rq   Nrz  r   r  Tr   )rQ  r  r   r^   )r  r  r  r   r  r   r   r   rI   r  r   r  rA   rq   )	r   rQ  r  r  model_value_router_model_name_model_valuepotential_wildcard_modelsr   s	            rf   ru  Router.get_model_list:  sR    4&&9;O%&&t'@'@J'@'WXt011040F0F0L0L0N,K!+s332=*#K66'@'O;'O'1T9$1=g1F. #**11'9 2  1O& ?#q(,0,?,?,E,Ej,Q),8#**;TU;Ta,1q1;TU !??2&&"" Vs   E c                     SSK Jn  U" [        5      nU R                  US9nU(       aJ  U HD  nUR	                  S0 5      R	                  S/ 5       H  nUS   nX6   R                  U5        M     MF     U$ )zF
If model_name is provided, only return access groups for that model.
r   r   rz  r  r   rQ  )collectionsr   r   ru  rL  r   )r   rQ  r   r   rq   r   groups          rf   get_model_access_groupsRouter.get_model_access_groupsi  sv     	,#D)((J(?
UU<488"ME!"<J!(//
; N  
 re   c                     [        U 5      n0 n/ SQnU HN  nXA;   a  X   X$'   US:X  d  M  U R                  S:X  d  M(  U R                  R                  R	                  5       X$'   MP     U$ )z
Get router settings method, returns a dictionary of the settings and their values.
For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after
r   r   r   r   r|   r~   r   r   r   r   r   r   r   )varsr   r  r  rY  )r   	_all_vars_settings_to_returnvars_to_includevars        rf   get_settingsRouter.get_settings|  su    
 J	 
 #C+4>#(..))-DD+/+D+D+Q+Q+V+V+X#( # #"re   c                    / SQn/ SQnU R                  5       nU H  nXR;   ac  XS;   a  [        X   5      n[        XU5        M(  US:X  a-  US   X   :w  a"  U R                  X   UR	                  S0 5      S9  [        XX   5        Mk  [
        R                  " SR                  U5      5        M     [
        R                  " SU R                  5        35        g)	z
Update the router settings.
r  )r~   r|   r   r   r   r   r   r   zSetting {} is not allowedzUpdated Router settings: N)r  r   r  r   rL  r   r   r   )r   r0  _allowed_settings_int_settings_existing_router_settingsr  _casted_values          rf   update_settingsRouter.update_settings  s    


 %)$5$5$7!C''$'$4MD}5 1156HIV[X22-3[28** 73 3  Dv{3%++,G,N,Ns,ST' ( 	##&?@Q@Q@S?T$UVre   c                    US   S   n[        U5      nUS:X  a.  SR                  U5      nU R                  R                  USUS9nU$ US:X  a  UR	                  S5      SL aU  U S	3nU R                  R                  USUS9nUc0   [
        R                  " XS
9  U R                  R                  USUS9nU$ U S3nU R                  R                  USUS9nUc0   [
        R                  " XS
9  U R                  R                  USUS9nU$ UR	                  S5      SL aQ  U S3nU R                  R                  XeS9nUc.   [
        R                  " XS
9  U R                  R                  XeS9nU$ U S3nU R                  R                  XeS9nUc.   [
        R                  " XS
9  U R                  R                  XeS9nU$ )at  
Returns the appropriate client based on the given deployment, kwargs, and client_type.

Parameters:
    deployment (dict): The deployment dictionary containing the clients.
    kwargs (dict): The keyword arguments passed to the function.
    client_type (str): The type of client to return.

Returns:
    The appropriate client based on the given client_type and kwargs.
r  r  r  z{}_max_parallel_requests_clientTr  r#  rh  r  rX  _stream_async_clientr  _async_client_stream_client)r  rh  _client)r   r   r   r$  rL  r$   r  )r   r#  r0  r  r  rh  	cache_keyr>  s           rf   rK  Router._get_client  sP    l+D1+LV+T119@@JIZZ))$AQ * F MG#zz(#t+'j(<=	--!dEU .  > -7704 "ZZ11%#')9 2 F
 'j6	--!dEU .  > -7704 "ZZ11%#')9 2 F
 zz(#t+'j7	--! .  > -7704 "ZZ11% 2 F 'j0	--! .  > -7704 "ZZ11% 2 F re   c           	      &
   [         R                  " SU 35        [        R                  " U5      n/ n [        R
                  " US9nSn	Sn
Sn[        U5      n[        5       nUR                  S5      nU SU 3nU R                  R                  US	US
9=(       d    0 n[        U5       GH%  u  nn UR!                  S0 5      R!                  SS5      nUc"  UR!                  S0 5      R!                  SS5      nU R#                  UUS9nU=(       d"    UR!                  S0 5      R!                  SS5      n[%        U[&        5      (       ac  UR!                  SS5      bP  [%        US   [(        5      (       a8  UUS   :  a/  UR+                  U5        S	n	U
SR                  UUS   U5      -  n
GM  UR!                  S0 5      nUR!                  S0 5      R!                  SS5      nU R                  R                  US	US
9=(       d    Sn[%        U[&        5      (       a  U R.                  S:w  a  UR!                  US5      UU'   [1        UUU   5      n[%        U[&        5      (       aJ  UR!                  SS5      b7  [%        US   [(        5      (       a  US   U::  a  UR+                  U5        S	nGM  UbR  UR!                  S5      b@  UR!                  S5      nUb,  [3        [5        S!0 UD6US9(       d  UR+                  U5        GMW  Uc  GM]  [        R6                  SL d  GMs  [        R8                  " U[5        S!0 UD6S9u  nn  n[        R:                  " UUS9nUc  GM  [        R<                  R?                  US9nS/nURA                  5        H@  u  nn UU;  d  M  UU;   d  M  [         R                  " SU 35        UR+                  U5        MB     GM(     [C        U5      [C        U5      :X  a9   US	L a	  [E        US9eU	S	L a%  [        RF                  " SR                  U
5      USS 9e[C        U5      S:  a#  [I        U5       H  nURK                  U5        M     [C        U5      S:  a  [        R<                  RM                  U5      nU$ ! [         a:  n[         R                  " SR                  [        U5      5      5        Us SnA$ SnAff = f! [         a9  n[         R,                  " SR                  [        U5      5      5         SnAGN`SnAff = f)"a  
Filter out model in model group, if:

- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
- filter models above rpm limits
- if region given, filter out models not in that region / unknown region
- [TODO] function call and model doesn't support function calling
z2Starting Pre-call checks for deployments in model=)r,  zllitellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {}NFrB  r  z:rpm:Tr(  r  r  r   )r#  r  r   r  z%Model={}, Max Input Tokens={}, Got={}zAn error occurs - {}r  r   r   r  allowed_model_region)r   r0  r  r  )passed_paramsresponse_formatz1INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=rZ  z~litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.
{}rC  r^   )'r   r   r   r   r   token_counterr'  r  r   r   r   rU   r  r   r$  r  rL  r  r   r   r   r   ra  r   maxrV   rC   drop_paramsrS   r  r  get_non_default_paramsr  r  rK   r  reversedrI  _get_order_filtered_deployments)!r   r   r   r,  rk  _returned_deploymentsinvalid_model_indicesinput_tokensr)  _context_window_error_potential_error_str_rate_limit_errorrh  r  r  r  model_group_cacher  r#  r  r  re  r  current_request_cache_localcurrent_requestr0  r^  r  r  non_default_paramsspecial_paramsr  r  s!                                    rf   _pre_call_checksRouter._pre_call_checks(  st     	##@H	
 !%.A B "	)"00(CL !&!!<^L W-G5 01JJ  ?O !   	 	  ))>?OCW'^^L"=AA,PTU
%!+0@"!E!I!I$d"J "77)u 8 
 # jnn5Er&J&N&NT'
 z400"'94@L #:.@#A3GG(:6H+II-44S904-,CJJ %z2D'E|,
 ! )nn-=rBO!~~lB7;;D"EH 

$$ TDT %   	 ( ,d33))-EE.?.C.CHa.P!(+"%/1B81L#
 55'++E48D #?5#93??+E2oE-44S9,0)  *"&&'=>J'5'9'9:P'Q$'3,'5'H'H-A .44S9  )g.A.AU.J3:3K3K0Q0Q40*Aq +2*M*M5H+' +2 *1)M)M&4 *N *& '8%8N 2 8 8 :1$;;^@S177"STUSV W 288= !;M  @\ $%-B)CC !D(/  '$.88 ^  e  e,  !#  $%) 56%))#. 7 $%)$+MM$Q$Q%%! %$G  	)!''~  F  FF
 )(	)l  W%//0F0M0McRSf0UVVWs6   R 1C7S
S
/S?S
S

T.TTc                     XR                   ;  a  gU R                   U   n[        U[        5      (       a  UnU$ US   nU$ )zt
Get the model from the alias.

Returns:
- str, the litellm model name
- None, if model is not in model group alias
Nr   )r   r   r   )r   r   _items      rf   _get_model_from_aliasRouter._get_model_from_alias  sL     ...&&u-eS!!E  'NEre   c                 `    U R                    Vs/ s H  o"S   S   U:X  d  M  UPM     sn$ s  snf )z&
Get the deployment by litellm model.
r   r   r  )r   r   r   s      rf    _get_deployment_by_litellm_model'Router._get_deployment_by_litellm_model  s0      ??T?a0@.A'.Je.S?TTTs   ++r8  c                 d   USL a  XR                  US94$ XR                  5       ;   aS  U R                  US9nUb&  UR                  R                  nXeR                  SS94$ [        SU SU R                   35      eU R                  US9nUb  UnXR                  ;  aZ  U R                  R                  US9nU(       a  X4$ U R                  b*  [        R                  " U R                  5      n	XS   S'   X4$ U R                  US	9n
[        U
5      S
:X  a  U R                  US9n
[         R"                  " SU
 35        [        U
5      S
:X  a%  [$        R&                  " SR)                  U5      USS9e[$        R*                  (       a'  U[$        R*                  ;   a  [$        R*                  U   nX4$ )a  
Common checks for 'get_available_deployment' across sync + async call.

If 'healthy_deployments' returned is None, this means the user chose a specific deployment

Returns
- str, the litellm model name
- List, if multiple models chosen
- Dict, if specific model chosen
TrZ  r  r   zBLiteLLM Router: Trying to call specific deployment, but Model ID :z6 does not exist in                     Model ID List: r   r   rz  r   zinitial list of deployments: zBYou passed in model={}. There is no 'model_name' with this string rB  rC  )rK  rM  r  r   r   r   r
  rH  rj   r   get_deployments_by_patternr   r   r   r  r  r   r   r   r  r   model_alias_map)r   r   r,  r  r8  r#  deployment_model_model_from_aliaspattern_deploymentsupdated_deploymentr   s              rf   r<  *Router._common_checks_available_deployment   s   $ $&??e?LLL((**,,e,<J%#-#<#<#B#B ')>)>D)>)QQQTUZT[ \$$($6$6#79 
 !66U6C(%E((("&"5"5"P"P #Q # #11 &&2%)]]++&" AF#34W=00 #7757I"#q("&"G"Ge"G"T##+,?+@A	
 "#q())\cc   ""u0G0G'G++E ))re   c                 B  #    U R                   S:w  aS  U R                   S:w  aC  U R                   S:w  a3  U R                   S:w  a#  U R                   S:w  a  U R                  UUUUUS9$  [        U5      nU R                  UUUUS9u  p[	        U[
        5      (       a  U$ [        XS9I S	h  vN n[        R                  " S
U 35        [        R                  " SU 35        U R                  UUS9nU R                  UUUb  [        [        [           U5      OS	UUS9I S	h  vN nU R                  (       a*  Ub'  U R                  U[        [        [            U5      UUS9n[#        U UUUS9I S	h  vN n[%        U5      S:X  a  ['        U UUS9I S	h  vN n	U	e[(        R(                  " 5       n
U R                   S:X  a2  U R*                  b%  U R*                  R-                  UUUUS9I S	h  vN nOU R                   S:X  a2  U R.                  b%  U R.                  R-                  UUUUS9I S	h  vN nOU R                   S:X  a3  U R0                  b&  U R0                  R-                  UUUUUS9I S	h  vN nO]U R                   S:X  a  [3        U UUS9$ U R                   S:X  a0  U R4                  b#  U R4                  R-                  UUS9I S	h  vN nOS	nUc  ['        U UUS9I S	h  vN n	U	e[        R6                  " SU SU R9                  U5       SU 35        [(        R(                  " 5       nX-
  n[:        R<                  " U R>                  RA                  [B        RD                  USUU
US95        U$  GN GNU GN	 GN GN GNV GN N N! [F         a  n[H        RJ                  " 5       nUbh  URM                  SS	5      nUbS  [N        RP                  " URR                  X4S9RU                  5         [:        R<                  " URW                  X5      5        UeS	nAff = f7f)z
Async implementation of 'get_available_deployments'.

Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps).
r   rp   r   r   r   )r   r,  r  r8  rk  r   r,  r  r8  r  Nzasync cooldown deployments: zcooldown_deployments: r   cooldown_deploymentsrS  r   r   r,  rk  )llm_router_instancer   rk  r   r   )rj  r   rh  r  r   r,  r  )r  r   r,  r  rk  rZ  r   r   r  r   $get_available_deployment for model: , Selected deployment:  for model: z2<routing_strategy>.async_get_available_deployments)rd  re  r   rh  rf  rg  r~  rI  ),r   rH  r   r<  r   r   r'   r   r   _filter_cooldown_deploymentsrV  r   r	   r6   r   rD  r   r!   r  r/   ro  r  async_get_available_deploymentsr!  r  r    rm   r   r*  rr  rs  r   rt  rM   ru  r'  rv  rw  rL  rM  rN  rO  rP  rL  )r   r   r,  r  r8  rk  rh  r   rX  ra  rf  r#  rg  ry  r)  rH  r  s                    rf   r|  %Router.async_get_available_deploymentN  s     !!%==%%)99%%)==%%)@@%%500!$7- 1  Z	@P)-)Q)Q!$7	 *R *&E -t44**)H(,* $  "''./C.DE "''*@AU@V(WX"&"C"C$7%9 #D #
 )-(N(N$7  + ./:-!1 )O 
) 
# **x/C&*&;&;(,T$Z9L(M%#1	 '< '# )@$(-$7	) # &'1,"E,0%5# 	
  J%%)AA,,8 22RR$),?!)#	 S    %%)==**6 00PP$),?!)#	 Q    %%)@@--9 33SS$),?!)#'5 T    &&*::%(,(;  %%5))5 //OO$),? P    "
!"E,0%5# 	
  !&&6ug=TUYUjUjkuUvTw  xD  EJ  DK  L yy{H -I''BB(//&R%5)% C 	 }$
#(#&0  	"+"6"6"8),001FM*$$*::5 eg''#99!Q G!	s   A$P'5N PN )M2*A6N  M5!AN /M80!N M;AN )M>*AN +N,AN .N/N P:N 	N
N "N	#BN 1P2N 5N 8N ;N >N N N N 	N 
PBPPPc                    U R                  UUUUS9u  p[        U[        5      (       a  U$ [        U5      n[	        XS9nU R                  UUS9nU R                  (       a  Ub  U R                  UUUUS9n[        U5      S:X  aG  U R                  US9n	U R                  R                  XS9n
[	        XS9n[        UU
U R                  US	9eU R                  S
:X  a'  U R                  b  U R                  R                  XS9nOU R                  S:X  a  [!        U UUS9$ U R                  S:X  a)  U R"                  b  U R"                  R                  UUUS9nOvU R                  S:X  a*  U R$                  b  U R$                  R                  UUUUS9nO<U R                  S:X  a*  U R&                  b  U R&                  R                  UUUUS9nOSnUca  [(        R*                  " SU S35        U R                  US9n	U R                  R                  XS9n
[	        XS9n[        UU
U R                  US	9e[(        R*                  " SU SU R-                  U5       SU 35        U$ )z2
Returns the deployment based on routing strategy
rV  r  rW  NrY  r   rz  )	model_idsrh  )r   r   r   cooldown_listr   r]  rp   r\  r   )r  r   rk  r   r[  r   r^  z, No deployment availabler_  r`  )r<  r   r   r   r)   ra  r   rD  r  rM  r   get_min_cooldownrJ   r   rm   get_available_deploymentsr    r  rn   r  r   r   r*  )r   r   r,  r  r8  rk  r   rh  rX  re  _cooldown_time_cooldown_listr#  s                rf   rH  Router.get_available_deployment  s&    &*%M%M 3	 &N &
" )400&&+L,
  9$( 
 #?? 3!5 @ 
 &&8+?"&"7"7$7!-	 #8 # "#q(**e*<I!00AA# B N 7(,N ','+'B'B,	    L0T5J5J5V..HH! I J ""&66 "$($7  !!%<<))522LL!$7- M J !!%::%%1..HH!$7!	 I J !!%==((411KK!$7!	 L J J!&&6ug=VW **e*<I!00AA# B N 7(,N ','+'B'B,	  	""25'9PQUQfQfgqQrPss  AF  @G  H	
 re   rX  c                     / n[         R                  " SU 35        U H#  nUS   S   nXR;   d  M  UR                  U5        M%     U H  nUR                  U5        M     U$ )a  
Filters out the deployments currently cooling down from the list of healthy deployments

Args:
    healthy_deployments: List of healthy deployments
    cooldown_deployments: List of model_ids cooling down. cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]

Returns:
    List of healthy deployments
zcooldown deployments: r  r  )r   r   r   r  )r   r   rX  deployments_to_remover#  r  s         rf   ra  #Router._filter_cooldown_deployments  so     !###&<=Q<R$ST-J&|4T:M4%,,Z8 . 0J&&z2 0""re   c                      UR                  S0 5      R                  SS5      nUc  Ub  U R                  XB5        ggg! [         a,  n[        R                  " S[        U5       35         SnAgSnAff = f)z'
Tracks successful requests rpm usage.
r  r  Nz$Error in _track_deployment_metrics: )rL  r'  r'  r   r  r   )r   r#  rh  r4  r  r)  s         rf   r   Router._track_deployment_metrics  s    
	Y!~~lB7;;D$GH '&&  (    	Y!''*NsSTvh(WXX	Ys   9> 
A4"A//A4ra  c                 B    [        UUU R                  U R                  S9$ )N)ra  r  r   r   )"_get_num_retries_from_retry_policyr   r   )r   ra  r  s      rf   r.   (Router.get_num_retries_from_retry_policy  s)     2#%)%B%B**	
 	
re   c                 V   U R                   nUc  g[        U[        R                  5      (       a  UR                  b  UR                  $ [        U[        R
                  5      (       a  UR                  b  UR                  $ [        U[        R                  5      (       a  UR                  b  UR                  $ [        U[        R                  5      (       a  UR                  b  UR                  $ [        U[        R                  5      (       a  UR                  b  UR                  $ gg)z
BadRequestErrorRetries: Optional[int] = None
AuthenticationErrorRetries: Optional[int] = None
TimeoutErrorRetries: Optional[int] = None
RateLimitErrorRetries: Optional[int] = None
ContentPolicyViolationErrorRetries: Optional[int] = None
N)r   r   r   r  BadRequestErrorAllowedFailsr  AuthenticationErrorAllowedFailsr  TimeoutErrorAllowedFailsr  RateLimitErrorAllowedFailsrP  'ContentPolicyViolationErrorAllowedFails)r   ra  r   s      rf   get_allowed_fails_from_policy$Router.get_allowed_fails_from_policy  s     >B=V=V' y'"9"9::$@@L'CCCy'"="=>>$DDP'GGGy'//22$==I'@@@y'"8"899$??K'BBBy'"E"EFF$LLX'OOO Y Gre   c                 F   SSK Jn  U R                  c  g U R                  nU" UR                  S/UR                  S9nX0l        [        R                  R                  U5        [        R                  R                  UR                  5        [        R                  " S5        g )Nr   )SlackAlertingslack)alerting_thresholdalertingdefault_webhook_urlz2[94m
Initialized Alerting for litellm.Router[0m
)1litellm.integrations.SlackAlerting.slack_alertingr}  r   r  webhook_urlslack_alerting_loggerr   r  r   r   !response_taking_too_long_callbackr   r   )r   r}  router_alerting_config_slack_alerting_loggers       rf   r   Router._initialize_alerting  s    S'151E1E!.5HHY 6 B B"
 &<"  !78  ''"DD	
 	""H	
re   CustomRoutingStrategyc                 `    [        U SUR                  5        [        U SUR                  5        g)z
Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router

Use this to set your custom routing strategy

Args:
    CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase
rH  r|  N)r  rH  r|  )r   r  s     rf   set_custom_routing_strategy"Router.set_custom_routing_strategy  s4     	&!::	

 	,!@@	
re   c                 N    S [         l        U R                  R                  5         g r   )r   r   flush_cacher  s    rf   r  Router.flush_cache  s    

 re   c                     / [         l        / [         l        / [         l        / [         l        S U l        U R                  5         g r   )r   r   r   r   r   r   r  r  s    rf   resetRouter.reset  s6    #% *,'#% *,' re   )Ar  r   r   r  r   r  r  r  r   r   r   r   r  rr   r   rk   r   ry   r   r   r   r   r   r   r   r   r{   r   r   r   r   r   r   r   r   r   r   rm   r!  r  rn   r  r}   r   r   rq   rj   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r~   r   )F)FFF)T)r  r  )NNNNr   )NNNNN)NNF)NNFN)r_   r`   ra   rb   rj   r	   __annotations__rk   r   r  rl   r   tenacityrm   r   rn   r   rH   r   rA   r   r   r   r>   r   r  floatr
   rI   rF   r=   rE   rB   r<   r   r   r   r   rL   r   r*  rR   rP   r5  r1  r   r[  rn  r2  r  rJ  r  r  r  r  r  rp  r  r   r  r  r9   r  r  r  r%  r#  r*  r1  r5  r4  r>  r=  rQ   rE  rC  rN  rM  r8   rU  rT  r7   rf  re  ro  r|  r   r  r   r   r  rq  r  r+  r  r  r'  r  r3  r  r  r   r   r   r   r  rY   r'  r0  r4  rO  r?  r   r  rN  LiteLLMLoggingr  r6   rV  ra  r@   ro  rk  r   r   rm  r  r  r  r  r  r  r  r  r-  rD   r  r  r  r  r  rM  r  r{  ru  r  r  r%  rK  rD  rH  rK  r<  r|  rH  ra  r  r.   rz  r   r?   r  r  r  rd   r^   re   rf   rh   rh      s   K&+OXd^+&11H:>h67>:>h67> ;?#'$($((,*/ ,0*.%) #' 7;!06 )+)+ ',%*     ,0 DH&(DH48 "#WY
$*+T$sCx.-AAB
Y
 $$78Y
 C=Y
 SMY
 SMY
 !Y
 "$Y
 Y
 !K
Y
" #Y
& #5/'Y
( #3-)Y
, c]-Y
.  
/Y
4 %5Y
6 !)!
7Y
< (0}=Y
> ?Y
@ _-AY
B $I
CY
H IY
J #'KY
L #'MY
N $eC!::;;<
OY
T !%UY
V #WY
X YY
Z +t#$
[Y
` #'#
aY
f  
gY
l '
mY
r  
sY
x $D>yY
z "&
{Y
J #++@"AKY
L  $MY
N !))@ AOY
P ".1QY
R "*!"
SY
X 
YY
v	F$ 8(01F(G8"? %os&: ;?TX?B4 ($(c3h$8	}11	2&@@$(c3h$8@	}11	2@H $(c3h$8BI$-	 
 SX$(c3h$8BI%.	 
 in$(c3h$8BGPTW^_dWeHeBf	"M1	2  JO(($(c3h$8(BF(T{{$(c3h$8{	}11	2{z	IS 	I$ 	I4 	I
$ 
$$ 
$G Gt GPT G( t 44 t uczAR8S "Z#S	Z# T#s(^,d4S#X3G.HHIZ#x++$(d38n)=$>+^ $(c3h$8BI$-	  SX$(c3h$8BI%.	  	K,K, tCH~&K, 	K,b bg$(c3h$8DGQXY^Q_	 
 $(c3h$8DGQXY]Q^	  << tCH~&< 	<|s 3 , ,C ,\c # *Es E3 EN( (3 (TC) CC CJY3 Ys Y3 Yv3 *(C (Tc H $)&+#(## # 4.	#
 d^# 4.#R $)&+#(  4.	
 d^ 4.8CS C# CR $)&+#(  4.	
 d^ 4.:CS C CR $)	 S$Y 4.	 
$6c4i 0 6 6x $(	 S$Y 4.	 
2CuS$Y'7 C CL 
	0OO 
	Ob 
	0UU 
	Un? 
?B//f
1#
1 :F# 56: EI(,	
#
 &g.?&@A
 %	
,N%f &*$(373711 c]1 D>	1
 #+4.1 #+4.1f}%~  :>)1#8 /3*.3737,0?? &d^? "$	?
 #+4.? #+4.? $D>?B%H &*$S$s)^,-$ c]$ 
$s)		$< /3*.// / 	/
 &d^/ "$/ 
sEz	/fPd! 
#!F< 
<| 
+3C= 
D     t  D 4<TN	> (,	::  S/:  }	:
 
:x ##$1#;?#	#J5c 5Xd^ 5.55,4TN5	tDz4:%	&5<54 5( 15	;; #4.; n-	;F *.041,1, "$Z1, 4 012	1,
 #4.1, !1, n-1,f'c '4 '8// / 	/
 / 
*	/b&z &d &P+A +AZb* b bH 8L 2 J  8J;O  DC HZ,@ .s x
/C " #	*	$ EI58>B	 
 58>A	  !	KTNK !K SM	K
 
KZ $ 
:# 
:(4. 
:A A >AA 	.	!A F
 
8P 
>G$G$	x}hsm+	,G$R cSVh 6 ;?""*23-"	"H c " =A,4SM	!	"(c   +/-"3--	$*+	,-` +/"3-	c49n	&#>0WdY@ *.}%}% "}% tCH~&	}%
 !}%~3 8C= &Uc Ud U 48,0.3L*L* 4S#X/0L* c4i()	L*
 &d^L* 
sE$*%%	&L*b 48,0.3)-uu 4S#X/0u c4i()	u
 &d^u !ut 48,0.3)-{{ 4S#X/0{ c4i()	{
 &d^{ !{z##':#EI#Y#	d#8 FJY,4TNY& BF
"
19#
&Py &PP
0
%>
.!re   rh   )rr  r   enumr[  r  rY  r   rM  ro  rv  r  r  r   typingr   r   r   r   r	   r
   r   r   r   r   httpxr   r   pydanticr   typing_extensionsr   r   litellm.litellm_core_utils2litellm.litellm_core_utils.exception_mapping_utilsr   litellm._loggingr   litellm.caching.cachingr   r   r   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   *litellm.litellm_core_utils.litellm_loggingr   r  &litellm.router_strategy.budget_limiterr   "litellm.router_strategy.least_busyr   #litellm.router_strategy.lowest_costr   &litellm.router_strategy.lowest_latencyr   &litellm.router_strategy.lowest_tpm_rpmr   )litellm.router_strategy.lowest_tpm_rpm_v2r   &litellm.router_strategy.simple_shuffler    )litellm.router_strategy.tag_based_routingr!    litellm.router_utils.batch_utilsr"   r#   /litellm.router_utils.client_initalization_utilsr$   #litellm.router_utils.cooldown_cacher%   &litellm.router_utils.cooldown_handlersr&   r'   r(   r)   r*   ,litellm.router_utils.fallback_event_handlersr+   r,   r-   *litellm.router_utils.get_retry_from_policyr.   rr  !litellm.router_utils.handle_errorr/   r0   Dlitellm.router_utils.pre_call_checks.prompt_caching_deployment_checkr1   >litellm.router_utils.router_callbacks.track_deployment_metricsr2   r3   litellm.schedulerr4   r5   litellm.types.llms.openair6   r7   r8   r9   litellm.types.routerr:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   litellm.types.servicesrM   litellm.types.utilsrN   r  rO   litellm.utilsrP   rQ   rR   rS   rT   rU   rV   &router_utils.pattern_match_deploymentsrX   opentelemetry.tracerY   _SpanEnumr[   rh   r^   re   rf   <module>r     s.              #       &  ! 9 " 2 H H ; U P G F H N J P A M U =  
 2 T T     * 0 9 6   G1DD$)) OV OVre   