
    ^hY                         S SK r S SKJrJr  S SKJrJrJrJrJrJ	r	  S SK
r
S SK
JrJrJr  S SKJr  S SKJr  S SKJr  S SKJr  \(       a	  S S	KJr  \rO\r " S
 S\5      r " S S\5      rg)    N)datetime	timedelta)TYPE_CHECKINGAnyDictListOptionalUnion)ModelResponsetoken_counterverbose_logger)	DualCache)CustomLogger)!_get_parent_otel_span_from_kwargs)LiteLLMPydanticObjectBase)Spanc                   @    \ rS rSr% Sr\\S'   Sr\\S'   Sr\	\S'   Sr
g	)
RoutingArgs   i  ttlr   lowest_latency_buffer
   max_latency_list_size N)__name__
__module____qualname____firstlineno__r   float__annotations__r   r   int__static_attributes__r       `/home/james-whalen/.local/lib/python3.13/site-packages/litellm/router_strategy/lowest_latency.pyr   r      s"    C#$5$!#3#r#   r   c                   v   \ rS rSr% Sr\\S'   Sr\\S'   Sr	\\S'   0 4S\
S\S	\4S
 jjrS rS rS r    SS\S\S\\\\\4         S\\\\4      S\\   S\\   4S jjr   SS\S\S\\\\\4         S\\\\4      S\\   4
S jjr   SS\S\S\\\\\4         S\\\\4      S\\   4
S jjrSrg)LowestLatencyLoggingHandler   F	test_flagr   logged_successlogged_failurerouter_cache
model_listrouting_argsc                 <    Xl         X l        [        S0 UD6U l        g )Nr   )r+   r,   r   r-   )selfr+   r,   r-   s       r$   __init__$LowestLatencyLoggingHandler.__init__!   s     )$'7,7r#   c                      US   R                  S5      c  g US   S   R                  SS 5      nUS   R                  S0 5      R                  SS 5      nUb  Uc  g [        U[        5      (       a  [        U5      n U S3n[        R
                  " 5       R                  S5      n[        R
                  " 5       R                  S5      n	[        R
                  " 5       R                  S	5      n
U S
U	 S
U
 3nXC-
  nS nUR                  SS 5      b  US   SL a  UR                  SU5      U-
  nUnS nSn[        U[        5      (       ac  [        USS 5      nUbS  UR                  nUR                  n[        UR                  5       U-  5      nUb  [        UR                  5       U-  5      n[        U5      nU R                  R                  UUS9=(       d    0 nUU;  a  0 UU'   [!        UU   R                  S/ 5      5      U R"                  R$                  :  a%  UU   R'                  S/ 5      R)                  U5        O,UU   S   S U R"                  R$                  S-
   U/-   UU   S'   Ub  [!        UU   R                  S/ 5      5      U R"                  R$                  :  a%  UU   R'                  S/ 5      R)                  U5        O,UU   S   S U R"                  R$                  S-
   U/-   UU   S'   UUU   ;  a  0 UU   U'   UU   U   R                  SS5      U-   UU   U   S'   UU   U   R                  SS5      S-   UU   U   S'   U R                  R+                  UUU R"                  R,                  S9  U R.                  (       a  U =R0                  S-  sl        g g ! [2         a8  n[4        R6                  " SR9                  [        U5      5      5         S nAg S nAff = f)Nlitellm_paramsmetadatamodel_group
model_infoid_map%Y-%m-%d%H%M-streamTcompletion_start_timer   usagekeyparent_otel_spanlatency   time_to_first_tokentpmrpmrA   valuer   `litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {})get
isinstancer!   strr   nowstrftimer   getattrcompletion_tokenstotal_tokensr   total_secondsr   r+   	get_cachelenr-   r   
setdefaultappend	set_cacher   r(   r)   	Exceptionr   	exceptionformat)r/   kwargsresponse_obj
start_timeend_timer5   r7   latency_keycurrent_datecurrent_hourcurrent_minuteprecise_minuteresponse_ms!time_to_first_token_response_timefinal_valuerE   rR   _usagerQ   rB   request_count_dictes                         r$   log_success_event-LowestLatencyLoggingHandler.log_success_event(   sw   F	 &'++J7?$%56zBFF!4 ,-11,CGGdS&"*C((RB
	 "-T2'||~66zB'||~66t<!)!8!8!>$0><..AQ!R)1)>IM1::h-9fX>NRV>V 

#:HE
R 6 8C7;# lM::$\7DAF),2,D,D)'-':':&+'557:KK' =H27 A O O Q"3!43/ $EV#L %%//':J 0   	 # //-/&r* *2.229bAB''==> 'r*55iDKKKX8J28N!9C))??!C9EHS}9U&r*95
 '2.r2667LbQR++AAB +2.9912 &!45 /r23HIId//EEI 0 +2.1 ");B)??=?&r*>: 'r*>:>>uaH<W #2&~6u= 'r*>:>>uaH1L #2&~6u= !!++#+=4CTCTCXCX , 
 >>''1,' " 	$$ryyF
 	s$   N' AN' MN' '
O)1.O$$O)c                   #     UR                  SS5      n[        U[        R                  5      (       Gan  US   R                  S5      c  gUS   S   R                  SS5      nUS   R                  S0 5      R                  SS5      nUb  Uc  g[        U[        5      (       a  [        U5      n U S3nU R                  R                  US	9I Sh  vN =(       d    0 n	Xy;  a  0 X'   [        X   R                  S
/ 5      5      U R                  R                  :  a$  X   R                  S
/ 5      R                  S5        O*X   S
   SU R                  R                  S-
   S/-   X   S
'   U R                  R                  UU	U R                  R                  S9I Sh  vN   gg N N! [         a8  n
[         R"                  " SR%                  [        U
5      5      5         Sn
A
gSn
A
ff = f7f)zB
Check if Timeout Error, if timeout set deployment latency -> 100
rZ   Nr3   r4   r5   r6   r7   r8   rA   rC   g     @@rD   rH   rJ   )rK   rL   litellmTimeoutr!   rM   r+   async_get_cacherU   r-   r   rV   rW   async_set_cacher   rY   r   rZ   r[   )r/   r\   r]   r^   r_   
_exceptionr5   r7   r`   ri   rj   s              r$   async_log_failure_event3LowestLatencyLoggingHandler.async_log_failure_event   s    >	K6J*goo66*+//
;C"()9"::"F"J"J%t#K   0155lBGKKDRVWB"*bj#B,, W
	 &1M"6K"//??K?PPVTV ' 313*. .266y"EF++AAB +.99)RHOOPVW<N<R%=GD--CCaG=ILR8=T*.y9 ++;;'0 --11 <    1 Q"  	$$ryyF
 	sn   G.AF) G.AF) G.AF) F%CF) F'F) #G.%F) 'F) )
G+3.G&!G.&G++G.c                   #      US   R                  S5      c  g US   S   R                  SS 5      nUS   R                  S0 5      R                  SS 5      nUb  Uc  g [        U[        5      (       a  [        U5      n U S3n[        R
                  " 5       R                  S5      n[        R
                  " 5       R                  S5      n	[        R
                  " 5       R                  S	5      n
U S
U	 S
U
 3nXC-
  nS nUR                  SS 5      b  US   SL a  UR                  SU5      U-
  nUnSnS n[        U[        5      (       ac  [        USS 5      nUbS  UR                  nUR                  n[        UR                  5       U-  5      nUb  [        UR                  5       U-  5      n[        U5      nU R                  R                  UUSS9I S h  vN =(       d    0 nUU;  a  0 UU'   [!        UU   R                  S/ 5      5      U R"                  R$                  :  a%  UU   R'                  S/ 5      R)                  U5        O,UU   S   S U R"                  R$                  S-
   U/-   UU   S'   Ub  [!        UU   R                  S/ 5      5      U R"                  R$                  :  a%  UU   R'                  S/ 5      R)                  U5        O,UU   S   S U R"                  R$                  S-
   U/-   UU   S'   UUU   ;  a  0 UU   U'   UU   U   R                  SS5      U-   UU   U   S'   UU   U   R                  SS5      S-   UU   U   S'   U R                  R+                  UUU R"                  R,                  S9I S h  vN   U R.                  (       a  U =R0                  S-  sl        g g  GN N/! [2         a8  n[4        R6                  " SR9                  [        U5      5      5         S nAg S nAff = f7f)Nr3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   Tr>   r   r?   )rA   rB   
local_onlyrC   rD   rE   rF   rG   rH   z\litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - {})rK   rL   r!   rM   r   rN   rO   r   rP   rQ   rR   r   rS   r   r+   rq   rU   r-   r   rV   rW   rr   r   r(   r)   rY   r   rZ   r[   )r/   r\   r]   r^   r_   r5   r7   r`   ra   rb   rc   rd   re   rf   rg   rR   rE   rh   rQ   rB   ri   rj   s                         r$   async_log_success_event3LowestLatencyLoggingHandler.async_log_success_event   s    G	 &'++J7?$%56zBFF!4 ,-11,CGGdS&"*C((RB

 "-T2'||~66zB'||~66t<!)!8!8!>$0><..AQ!R)1)>IM1::h-9fX>NRV>V 

#:HE
R 6 8C 7;#lM::$\7DAF),2,D,D)'-':':&+'557:KK' =H27 A O O Q"3!43/ $EV#L ++;;')9#' <   
  # //-/&r* *2.229bAB''==> 'r*55iDKKKX8J28N!9C))??!C9EHS}9U&r*95
 '2.r2667LbQR++AAB +2.9912 &!45 /r23HIId//EEI 0 +2.1 ");B)??=?&r*>: 'r*>:>>uaH<W #2&~6u= 'r*>:>>uaH1L #2&~6u= ''77#+=4CTCTCXCX 8   
 >>''1,' "uj  	$$nuuF
 	sk   PN? PAN? PE9N? N:F4N? N=*N? 8P:N? =N? ?
P	.O<7P<PPNr5   healthy_deploymentsmessagesinputrequest_kwargsri   c                 "   0 n[        S5      n[        R                  " 5       R                  S5      n	[        R                  " 5       R                  S5      n
[        R                  " 5       R                  S5      nU	 SU
 SU 3nSnUc  gUnU H#  nUS   S   U;  d  M  S	S
/US
S
S.0XS   S   '   M%      [	        X4S9nUR                  5       n[        R                  " [        U5      [        U5      5      n[        U5      n/ nUR                  5        GHU  u  nnSnU H  nUUS   S   :X  d  M  UnM     Uc  M'  UR                  SS5      =(       d]    UR                  S0 5      R                  SS5      =(       d4    UR                  S0 5      R                  SS5      =(       d    [        S5      nUR                  SS5      =(       d]    UR                  S0 5      R                  SS5      =(       d4    UR                  S0 5      R                  SS5      =(       d    [        S5      nUR                  S	/ 5      nUR                  S/ 5      nUR                  U0 5      R                  SS
5      nUR                  U0 5      R                  SS
5      nSnUbP  UR                  SS5      b=  US   SL a5  [        U5      S
:  a&  U H  n[        U[         5      (       d  M  UU-  nM!     O%U H  n[        U[         5      (       d  M  UU-  nM!     U[        U5      -  nUR                  S0 5      R                  SS5      n U b  UUU '   UU-   U:  d	  US-   U:  a  GMB  UR                  UU45        GMX     [        U5      S
:X  a  g[        US S9n!U!S
   S   nU R                   R"                  U-  n"U! V#s/ s H  n#U#S   UU"-   ::  d  M  U#PM     n$n#[        R$                  " U$5      n%U%S
   nUb  SU;   a  UUS   S'   U$ ! [
         a    S
n GNIf = fs  sn#f )z>Common logic for both sync and async get_available_deploymentsinfr9   r:   r;   r<   Nr6   r7   rC   r   )rF   rG   )r{   textrF   r3   rG   rE   g        r=   Tapi_base rD   c                     U S   $ )NrD   r   )xs    r$   <lambda>HLowestLatencyLoggingHandler._get_available_deployments.<locals>.<lambda>  s    1r#   rn   r4   _latency_per_deployment)r   r   rN   rO   r   rY   itemsrandomsamplelistrU   dictrK   rL   rW   sortedr-   r   choice)&r/   r5   rz   r{   r|   r}   ri   r   lowest_latencyra   rb   rc   rd   
deploymentall_deploymentsdinput_tokens_items_all_deploymentspotential_deploymentsitemitem_map_deploymentm_deployment_tpm_deployment_rpmitem_latencyitem_ttft_latencyitem_rpmitem_tpmtotal_call_latency_deployment_api_basesorted_deploymentsbufferr   valid_deploymentsrandom_valid_deployments&                                         r$   _get_available_deployments6LowestLatencyLoggingHandler._get_available_deployments  sP    #%u||~..z:||~..t4!006(><..9IJ
%,$At$O;s"Aa$8:, 56 %	((GL
 !&&(!==fs6{C/0 !#-335ND(K(1\?400"#K ) " t,  ??#3R8<<UDI ??<488E  <	  t,  ??#3R8<<UDI ??<488E  <	  $<<	26L (-BB G||NB7;;E1EH||NB7;;E1EH E*"&&x6B"8,4)*Q.%6M!-77. &7 &2M!-77. &2 !3|#44L $/??3CR#H#L#LB$  $/@L'(<= <'/9a</1%,,k<-HIC 6F $%* $$9~N ,A.q1 ""88>I *
)!QqT^f5L-LA) 	 

 #)--0A"B,Q/
%**F ( :&) M  	L	t
s   6	O9 /PP9P	P	c                    #    U S3n[        U5      nU R                  R                  XgS9I S h  vN =(       d    0 nU R                  UUUUUU5      $  N#7f)Nr8   r@   )r   r+   rq   r   	r/   r5   rz   r{   r|   r}   r`   rB   ri   s	            r$   async_get_available_deployments;LowestLatencyLoggingHandler.async_get_available_deployments  s      %T*+L,
 ##33 4    	 	 ..
 	
s   -AA$Ac                     U S3n[        U5      nU R                  R                  XgS9=(       d    0 nU R                  UUUUUU5      $ )z.
Returns a deployment with the lowest latency
r8   r@   )r   r+   rT   r   r   s	            r$   get_available_deployments5LowestLatencyLoggingHandler.get_available_deployments/  sr     %T*+L,
 '' (   	 	 ..
 	
r#   )r,   r+   r-   )NNNN)NNN)r   r   r   r   r(   boolr    r)   r!   r*   r   r   r   r0   rk   rt   rx   rM   r	   r   r   r
   r   r   r   r"   r   r#   r$   r&   r&      s   ItNCNC OQ8%8378GK8IVBHJ` 48,0)--1LL "L 4S#X/0	L
 c4i()L !L %TNLd 48,0)-

 "
 4S#X/0	

 c4i()
 !
D 48,0)-

 "
 4S#X/0	

 c4i()
 !
 
r#   r&   )r   r   r   typingr   r   r   r   r	   r
   ro   r   r   r   litellm.caching.cachingr   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   litellm.types.utilsr   opentelemetry.tracer   _Spanr   r&   r   r#   r$   <module>r      sS     ( B B  @ @ - ; U 91DD$+ $r
, r
r#   