
    ^h                     0   S SK r S SKrS SKJrJr  S SKJrJrJrJrJ	r	J
r
JrJr  S SKJr  S SKJr  S SKrS SKJrJr  S SKJr  S SKJr  S S	KJr  S S
KJr  S SKJrJr  S SKJ r J!r!  \(       a  S SK"J#r$  S SK%J&r'  \$r#\'r&O\r#\r& " S S\5      r( " S S\5      r)g)    N)datetime	timedelta)TYPE_CHECKINGAnyListLiteralOptionalTuple	TypedDictUnion)HTTPException)	BaseModel)	DualCacheModelResponseverbose_proxy_logger)(RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY)CustomLogger)!_get_parent_otel_span_from_kwargs)CurrentItemRateLimitUserAPIKeyAuth)get_key_model_rpm_limitget_key_model_tpm_limit)Span)InternalUsageCachec                   f    \ rS rSr% \\   \S'   \\   \S'   \\   \S'   \\   \S'   \\   \S'   Srg)	CacheObject!   current_global_requestsrequest_count_api_keyrequest_count_user_idrequest_count_team_idrequest_count_end_user_id N)__name__
__module____qualname____firstlineno__r	   dict__annotations____static_attributes__r$       f/home/james-whalen/.local/lib/python3.13/site-packages/litellm/proxy/hooks/parallel_request_limiter.pyr   r   !   s4    %d^+#D>)#D>)#D>)'~-r,   r   c                   X  ^  \ rS rSrS\4S jrS rS\S\S\	S\
S	\S
\S\S\\	   S\
S\S   S\\\\4      4S jrS\4S jr S#S\\
   S\4S jjr S#S\\
   S\\
   S\\
   S\\
   S\\
   S\\   S\4S jjrS\S\S\	S\
4S jrS rS rS\
S\S\\	   4S  jrS\	S\4U 4S! jjrS"rU =r$ )$!_PROXY_MaxParallelRequestsHandler)   internal_usage_cachec                     Xl         g Nr1   )selfr1   s     r-   __init__*_PROXY_MaxParallelRequestsHandler.__init__+   s    $8!r,   c                      [         R                  " U5        [        R                  (       a  [	        U5        g g ! [
         a     g f = fr3   )r   debuglitellmset_verboseprint	Exception)r5   print_statements     r-   print_verbose/_PROXY_MaxParallelRequestsHandler.print_verbose.   s=    	 &&7""o& # 		s   6: 
AAuser_api_key_dictcachedata	call_typemax_parallel_requests	tpm_limit	rpm_limitcurrentr    rate_limit_type)usercustomerteamvalues_to_update_in_cachec                   #    UcF  US:X  d  US:X  d  US:X  a  U R                  SU
 SU SU SU 3S9$ SSSS.nUR                  X45        g [        US	   5      U:  a7  US
   U:  a.  US   U:  a%  US	   S-   US
   US   S.nUR                  X45        g [        SSU
 SUS    SU SUS
    SU 3
S[	        U R                  5       5      0S9e7f)Nr   Hit limit for z). Current limits: max_parallel_requests: , tpm_limit: , rpm_limit: additional_details   current_requestscurrent_tpmcurrent_rpmrV   rW   rX     z1LiteLLM Rate Limit Handler for rate limit type = z'. Crossed TPM, RPM Limit. current rpm: z, rpm limit: z, current tpm: z, tpm limit: retry-afterstatus_codedetailheaders)raise_rate_limit_errorappendintr   strtime_to_next_minute)r5   rA   rB   rC   rD   rE   rF   rG   rH   r    rI   rM   new_vals                r-   check_key_in_limits5_PROXY_MaxParallelRequestsHandler.check_key_in_limits6   s    $ ?$)Y!^yA~22)77HHq  sH  rI  IV  W`  Va  an  ox  ny  (z 3   %&  G
 &,,.C-MN*+,/DD&2&2 %,,>$?!$C&}5&}5G
 &,,.C-MNJ?J[  \C  DK  LY  DZ  C[  [h  ir  hs  sB  CJ  KX  CY  BZ  Zg  hq  gr  s&D,D,D,F(GH s   CCreturnc                     [         R                  " 5       nU[        SS9-   R                  SSS9nX!-
  R	                  5       nU$ )NrT   )minutesr   )secondmicrosecond)r   nowr   replacetotal_seconds)r5   rl   next_minuteseconds_to_next_minutes       r-   rc   5_PROXY_MaxParallelRequestsHandler.time_to_next_minuteg   sI    lln Yq11::!QR:S #."3!B!B!D%%r,   rS   c           	      j    SnUb  US-   U-   n[        SSU 3S[        U R                  5       5      0S9e)zH
Raise an HTTPException with a 429 status code and a retry-after header
"Max parallel request limit reached rY   z#Max parallel request limit reached rZ   r[   )r   rb   rc   )r5   rS   error_messages      r-   r_   8_PROXY_MaxParallelRequestsHandler.raise_rate_limit_errors   sQ     =))C/2DDM89K8LM"C(@(@(B$CD
 	
r,   r   r!   r"   r#   parent_otel_spanc                    #    UUUUU/nU R                   R                  UUS9I S h  vN nUc  [        S S S S S S9$ [        US   US   US   US   US   S9$  N07f)N)keysrw   )r   r    r!   r"   r#   r   rT            )r1   async_batch_get_cacher   )	r5   r   r    r!   r"   r#   rw   ry   resultss	            r-   get_all_cache_objects7_PROXY_MaxParallelRequestsHandler.get_all_cache_objects   s      $!!!%
 11GG- H 
 

 ?(,&*&*&**.  $+AJ")!*")!*")!*&-aj
 	

s   %AA1Ac                 X  #    U R                  S5        UR                  nUR                  nUc  [        R                  nUc  0 nUR                  S0 5      R                  SS 5      n[        US[        R                  5      nUc  [        R                  n[        US[        R                  5      n	U	c  [        R                  n	/ n
S nUb|  SnU R                  R                  USUR                  S9I S h  vN nUc  SnX:  a  U R                  S	U S
U 3S9$ U R                  R                  USSUR                  S9I S h  vN   [        R                  " 5       R                  S5      n[        R                  " 5       R                  S5      n[        R                  " 5       R                  S5      nU SU SU 3nU R                  Ub  SOS Ub  U SU S3OS UR                   b  UR                    SU S3OS UR"                  b  UR"                   SU S3OS UR$                  b  UR$                   SU S3OS UR                  S9I S h  vN nUGb%  U SU S3nUS   nU R                  SU 35        U[        R                  :X  a)  U[        R                  :X  a  U	[        R                  :X  a  OUS:X  d  US:X  d  U	S:X  a"  U R                  S[&         SU SU SU SU	 3
S9$ Uc  SSSS.nU
R)                  UU45        Ov[+        US   5      U:  a8  US   U:  a/  US   U	:  a&  US   S-   US   US   S.nU
R)                  UU45        O,U R                  S[&         SU S U S!US    S"U	 S#US    S$3S9$ [-        U5      c  [/        U5      Gb  UR                  S%S 5      nU SU SU S3nU R                  R                  UUR                  S&9I S h  vN nS nS n[-        U5      n[/        U5      nUb0  U(       a  UR                  U5      nU(       a  UR                  U5      nUc  SSSS.nU
R)                  UU45        OUc  Ub  US   S-   US   US   S.nUb/  US   U:  a&  U R                  S'U S([&         SU S U S!US    S$3S9$ Ub/  US   U:  a&  U R                  S)U S([&         SU S*U S+US    S$3S9$ U
R)                  UU45        S nS nU(       a  Ub  UUS   -
  nUb  UUS   -
  nS,U 3US-U 3U0nSU;  a  0 US'   US   R1                  U5        UR                   nUbx  UR2                  nUR4                  nUc  [        R                  nUc  [        R                  nU SU S3nU R7                  UUUU[        R                  US.   UUUS/U
S09I S h  vN   UR"                  n U bx  UR8                  n!UR:                  n"U!c  [        R                  n!U"c  [        R                  n"U  SU S3nU R7                  UUUU[        R                  US1   UU!U"S2U
S09I S h  vN   UR$                  (       a  [        US3[        R                  5      n#[        US4[        R                  5      n$U#c  [        R                  n#U$c  [        R                  n$UR$                   SU S3nU R7                  UUUU[        R                  UUS5   U#U$S6U
S79I S h  vN   [<        R>                  " U R                  RA                  U
S8UR                  S995        g  GN$ GN GN GNL GN N NO7f):Nz)Inside Max Parallel Request Pre-Call Hookmetadataglobal_max_parallel_requestsrF   rG   Tkey
local_onlylitellm_parent_otel_spanrT   zHit Global Limit: Limit=z, current: rR   r   valuer   r   %Y-%m-%d%H%M-::::request_count)r   r    r!   r"   r#   rw   r    z	current: r   rO   z: z. max_parallel_requests: rP   rQ   rU   rV   rW   rX   z. tpm_limit: z, current_tpm z , rpm_limit: z current rpm rt   modelr   r   zHit TPM limit for model: z on zHit RPM limit for model: z. rpm_limit: z, current_rpm zlitellm-key-remaining-tokens-zlitellm-key-remaining-requests-r!   rJ   )rA   rB   rC   rD   rE   rH   r    rF   rG   rI   rM   r"   rL   end_user_tpm_limitend_user_rpm_limitr#   rK   )rA   rB   rC   rD   rE   r    rH   rF   rG   rI   rM   <   
cache_listttlr   )!r?   api_keyrE   sysmaxsizegetgetattrr1   async_get_cacherw   r_   async_increment_cacher   rl   strftimer   user_idteam_idend_user_idr   r`   ra   r   r   updateuser_tpm_limituser_rpm_limitre   team_tpm_limitteam_rpm_limitasynciocreate_taskasync_batch_set_cache)%r5   rA   rB   rC   rD   r   rE   r   rF   rG   rM   rd   _keyr   current_datecurrent_hourcurrent_minuteprecise_minutecache_objectsr    rH   _modeltpm_limit_for_modelrpm_limit_for_model_tpm_limit_for_key_model_rpm_limit_for_key_model_remaining_tokens_remaining_requests_remaining_limits_datar   r   r   r   r   r   r   r   s%                                        r-   async_pre_call_hook5_PROXY_MaxParallelRequestsHandler.async_pre_call_hook   s*
     	FG#++ 1 G G ($'KK!<D'+xx
B'?'C'C*D(
$ -{CKKH	I-{CKKH	I  	" #''31D,0,E,E,U,U):)K)K -V - '# '.*+'&F22)AB^A__j  lC  kD  (E 3  
 //EE#->-O-O	 F     ||~..z:||~..t4!006(><..9IJ+/+E+E 0; / & )2n-_= %,,8 %,,-R/?O %,,8 %,,-R/?O %00< %001N3C?S.??5 ,F ,
 &
8 '.ir.1A$Q! $$;<G7)45%4,,&!+yA~a22)78`7aacdkcl  mF  G\  F]  ]j  kt  ju  uB  CL  BM  (N 3   ()#$#$
 *002G1QRG./03HHM*Y6M*Y6 )00B(Ca(G#*=#9#*=#9
 *002G1QR22)78`7aacdkclly  {D  zE  ES  T[  \i  Tj  Sk  ky  zC  yD  DQ  RY  Zg  Rh  Qi  ij  (k 3   $$56B&'89EXXgt,F)2fXR'7G " !55EE)):)K)K F  G
 #'"&'>?P'Q$'>?P'Q$!+*B*F*Fv*N'+*B*F*Fv*N'()#$#$
 *002G1QR$04G4S )00B(Ca(G#*=#9#*=#9 (3.2EE66-FvhdS{R||~  @G  H  HU  Vi  Uj  jx  y@  AN  yO  xP  PQ  ,R 7   (3.2EE66-FvhdS{R||~  @G  H  HU  Vi  Uj  jx  y@  AN  yO  xP  PQ  ,R 7   .446KW5UV $"&&2(;gm>T(T%&2*=@V*V' 0x8:K1&:<O&"
 %#%Z ##$:; $++.==N.==N%!$%!$'.ir.1A$Q!**"3#&)kk%&=>&;(( &*C +     $++.==N.==N%!$%!$'.ir.1A$Q!**"3#&)kk%&=>&;(( &*C +     ((!(!#7" ")!#7" ")%([["!)%([[" %001N3C?S "
 **"3#&)kk&;%&AB,, **C +    	%%;;4):)K)K < 	
 	]'&
^^8Fs   C1\*3\4A\* \D\*	\
F
\*\ G\* \#!B\*'\&(B0\*\(?\*\*\* \*#\*&\*(\*c                 .
  #    SSK Jn  [        US9n U R                  S5        US   S   R	                  SS 5      nUS   S   S   nUS   S   R	                  S	S 5      n	US   S   R	                  S
S 5      n
UR	                  S5      nUS   S   R	                  S0 5      =(       d    0 nUb&  SnU R
                  R                  USSUS9I S h  vN   [        R                  " 5       R                  S5      n[        R                  " 5       R                  S5      n[        R                  " 5       R                  S5      nU SU SU 3nSn[        U[        5      (       a  UR                  R                  n/ nUb  U SU S3nU R
                  R                  UUS9I S h  vN =(       d    SSSS.n[        US   S-
  S5      US   U-   US   S-   S.nU R                  SU SU 35        UR!                  UU45        U" U5      nUb  Ub  SU;   d  SU;   a  U SU SU S3nU R
                  R                  UUS9I S h  vN =(       d    SSSS.n[        US   S-
  S5      US   U-   US   S-   S.nU R                  SU SU 35        UR!                  UU45        U	b  Sn[        U[        5      (       a  UR                  R                  nU	 SU S3nU R
                  R                  UUS9I S h  vN =(       d    SUSS.n[        US   S-
  S5      US   U-   US   S-   S.nU R                  SU SU 35        UR!                  UU45        U
b  Sn[        U[        5      (       a  UR                  R                  nU
 SU S3nU R
                  R                  UUS9I S h  vN =(       d    SUSS.n[        US   S-
  S5      US   U-   US   S-   S.nU R                  SU SU 35        UR!                  UU45        Ub  Sn[        U[        5      (       a  UR                  R                  nU SU S3nU R
                  R                  UUS9I S h  vN =(       d    SUSS.n[        US   S-
  S5      US   U-   US   S-   S.nU R                  SU SU 35        UR!                  UU45        U R
                  R#                  US US!9I S h  vN   g  GN GN> GN GN GNA N N! [$         a  nU R                  U5         S nAg S nAff = f7f)"Nr   )#get_model_group_from_litellm_kwargskwargsz5INSIDE parallel request limiter ASYNC SUCCESS LOGGINGlitellm_paramsr   r   user_api_keyuser_api_key_user_iduser_api_key_team_idrJ   user_api_key_metadataTr   r   r   r   r   r   r   r   rT   rU   rV   rW   rX   zupdated_value in success call: z, precise_minute: model_rpm_limitmodel_tpm_limitr   r   ))litellm.proxy.common_utils.callback_utilsr   r   r?   r   r1   r   r   rl   r   
isinstancer   usagetotal_tokensr   maxr`   r   r=   )r5   r   response_obj
start_timeend_timer   r   r   r   r   r   user_api_key_end_user_idr   r   r   r   r   r   r   rM   r    rH   rd   model_groupes                            r-   async_log_success_event9_PROXY_MaxParallelRequestsHandler.async_log_success_event  s    	
 7X7
 X	"VW+12B+CJ+O+S+S.,( ""23J?OL#)*:#;J#G#K#K&$  $**:#;J#G#K#K&$  (.zz&'9$ '(4889PRTU  " ,75//EE#-E	 F    $<<>22:>L#<<>2248L%\\^44T:N ,~Q|nAn=MNNL,66+11>> )+%'#nB~&6oF & !% 9 9 I I--E !J !   )*#$#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR
 >fEK(+%)>>(,AA $nB{m2n5E_U & !% 9 9 I I--E !J !   )*#$#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR
 $/ lM::#/#5#5#B#BL ,,B~.>oN & !% 9 9 I I--E !J !   )*#/#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR
 $/ lM::#/#5#5#B#BL ,,B~.>oN & !% 9 9 I I--E !J !   )*#/#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR
 (3 lM::#/#5#5#B#BL 00>2B/R & !% 9 9 I I--E !J !   )*#/#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9A#= ""5gY>PQ_P`a *002G1QR++AA4)A B   o8HBBB(
  	"q!!	"s   TB:S- SCS- S BS- ?S  B2S- 2S#3B2S- %S&&B2S- S)A;S- S+S- TS- S-  S- #S- &S- )S- +S- -
T7TTTTc                 T  #     U R                  S5        [        US9nUS   R                  S0 5      =(       d    0 nUR                  SS 5      nUR                  SS 5      nU R                  SU 35        Uc  g S[        US	   5      ;   a  g UbI  Sn	U R                  R                  U	S
US9I S h  vN   U R                  R                  U	SS
US9I S h  vN   [        R                  " 5       R                  S5      n
[        R                  " 5       R                  S5      n[        R                  " 5       R                  S5      nU
 SU SU 3nU SU S3nU R                  R                  UUS9I S h  vN =(       d    SSSS.n[        US   S-
  S5      US   US   S.nU R                  SU 35        U R                  R                  UUSUS9I S h  vN   g  GN+ GN
 Nl N! [         a8  n[        R                  " SR                  [        U5      5      5         S nAg S nAff = f7f)Nz(Inside Max Parallel Request Failure Hookr   r   r   r   r   zuser_api_key: rs   	exceptionTr   r   r   r   r   r   r   r   r   r   rT   r   rU   rV   rW   rX   zupdated_value in failure call: r   )r   r   z;Inside Parallel Request Limiter: An exception occurred - {})r?   r   r   rb   r1   r   r   r   rl   r   r   async_set_cacher=   r   r   format)r5   r   r   r   r   r   	_metadatar   r   r   r   r   r   r   r    rH   rd   r   s                     r-   async_log_failure_event9_PROXY_MaxParallelRequestsHandler.async_log_failure_event  s    O	IJ1@ % /044ZDJI+4==.,( %==>L~>?# 4s6+;N7OO 0;9D"77GG $'+5M H    33II  #'1I	 J     (||~66zB'||~66t<!)!8!8!>$0><..AQ!R $nB~&6oF & !% 9 9 I I--E !J !   )*#$#$  ),G4F,G!,KQ(O#*=#9#*=#9 ""%DWI#NO//??)-E	 @   U&   	 **MTTF 	s   H(A3G# 7H(8G# 
H(#G# .G/#G# GBG# 2G3A G# G!G# H(G# G# G# !G# #
H%-.H H( H%%H(r   c           	      $  #    SSK Jn  SSKJn  SSKJn   U" UUU R                  R                  SUR                  SS9I Sh  vN nUc  gUR                  5       $  N! [         a&  nUR                  " S[        U5      5         SnAgSnAff = f7f)	a  
Helper to get the 'Internal User Object'

It uses the `get_user_object` function from `litellm.proxy.auth.auth_checks`

We need this because the UserApiKeyAuth object does not contain the rpm/tpm limits for a User AND there could be a perf impact by additionally reading the UserTable.
r   r   )get_user_object)prisma_clientFN)r   r   user_api_key_cacheuser_id_upsertrw   proxy_logging_objz3Parallel Request Limiter: Error getting user object)litellm._loggingr   litellm.proxy.auth.auth_checksr   litellm.proxy.proxy_serverr   r1   
dual_cacherw   
model_dumpr=   r9   rb   )r5   r   rA   r   r   r   _user_id_rate_limitsr   s           r-   get_internal_user_object:_PROXY_MaxParallelRequestsHandler.get_internal_user_object  s      	:B<	)8+#'#<#<#G#G$!2!C!C"&* $  $+'2244$  	 &&Es1v 		sL   B,A AA 
BA BA 
B'BBBBc                   >#    UR                   n[        R                  " 5       R                  S5      n[        R                  " 5       R                  S5      n[        R                  " 5       R                  S5      nU SU SU 3nU SU S3n	U R                  R                  U	UR                  S9I Sh  vN n
SnSnSnSnU
bV  UR                  b  UR                  U
S	   -
  nUR                  nUR                  b  UR                  U
S
   -
  nUR                  n[        US5      (       a  [        US5      nOSnUb  [        U[        5      (       d  [        U[        5      (       a  [        U[        5      (       a  UR                  5       nUR                  S0 5      =(       d    0 nUb  UUS'   Ub  UUS'   Ub  UUS'   Ub  UUS'   [!        US0 UESU0E5        ["        TU ]I  XU5      I Sh  vN $ gg GN@ N	7f)z+
Retrieve the key's remaining rate limits.
r   r   r   r   r   r   r   NrX   rW   _hidden_paramsadditional_headerszx-ratelimit-remaining-requestszx-ratelimit-limit-requestszx-ratelimit-remaining-tokenszx-ratelimit-limit-tokens)r   r   rl   r   r1   r   rw   rG   rF   hasattrr   r   r   r)   r   r   setattrsuperasync_post_call_success_hook)r5   rC   rA   responser   r   r   r   r   r    rH   key_remaining_rpm_limitkey_rpm_limitkey_remaining_tpm_limitkey_tpm_limitr   _additional_headers	__class__s                    r-   r   >_PROXY_MaxParallelRequestsHandler.async_post_call_success_hook6  sG     $++||~..z:||~..t4!006(><..9IJ#*)2n-=_ M++;;)):)K)K <   	 26'+15'+ **6%//'-2HH ( !2 ; ; **6%//'-2HH ( !2 ; ;8-..$X/?@N!N%~y11ZPT5U5U.)44!/!:!:!<"0"4"45I2"N"TRT&2+ $$DE (DQ#$@A&2+ $$BC (BO#$>? M>M#79LM =  5 6V &5js%   B2G:5G56D8G:.G8/G:8G:r4   r3   ) r%   r&   r'   r(   r   r6   r?   r   r   r)   rb   ra   r	   r   r   r
   r   re   floatrc   r   r_   r   r   r   r   r   r   r   r   r+   __classcell__)r   s   @r-   r/   r/   )   s   9-? 9/)/ / 	/
 /  #/ / / $/  #/ !!;</ $(c3h#8/b
&U 
& 37
"*3-
	
, ,0$
!)#$
  (}$
  (}	$

  (}$
 $,C=$
 #4.$
 
$
Lr)r r 	r
 rh	b"HPd"" *" 
$	"HDD-;D Dr,   r/   )*r   r   r   r   typingr   r   r   r   r	   r
   r   r   fastapir   pydanticr   r:   r   r   r   r   litellm.constantsr   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   litellm.proxy._typesr   r   litellm.proxy.auth.auth_utilsr   r   opentelemetry.tracer   _Spanlitellm.proxy.utilsr   _InternalUsageCacher   r/   r$   r,   r-   <module>r     sr     
 ( W W W !   , 1 F ; U E
 1MD,D.) .Q Qr,   