
    ^hW                         S SK r S SKJr  S SKrS SKJrJr  S SKJrJ	r	  Sq
 " S S\5      rS\4S jr0 SS4S\S	\S
\S\S\4
S jjrS0 4S\S	\4S jjrS rg)    N)Callable)custom_promptprompt_factory)ModelResponseUsagec                   (   ^  \ rS rSrU 4S jrSrU =r$ )	VLLMError   c                    > Xl         X l        [        R                  " SSS9U l        [        R
                  " XR                  S9U l        [        TU ]!  U R                  5        g )NPOSTzhttp://0.0.0.0:8000)methodurl)status_coderequest)	r   messagehttpxRequestr   Responseresponsesuper__init__)selfr   r   	__class__s      ^/home/james-whalen/.local/lib/python3.13/site-packages/litellm/llms/vllm/completion/handler.pyr   VLLMError.__init__   sI    &}}F8MN;ULL	
    )r   r   r   r   )__name__
__module____qualname____firstlineno__r   __static_attributes____classcell__)r   s   @r   r	   r	      s    
 
r   r	   modelc                      SSK JnJn  [        c  U" U S9q[        U4$ ! [         a  n[        S[        U5      S9eS nAff = f)Nr   )LLMSamplingParamsr#   r   r   )vllmr%   r&   llm	Exceptionr	   str)r#   r%   r&   es       r   validate_environmentr.      sE    7,;E"CN"" 7As1v667s    
A<Amessagesmodel_responseprint_verboseoptional_paramsc
                 (    [        U S9u  qn
U
" S0 UD6nX;   a  Xp   n[        US   US   US   US9nO	[        XS9nUR                  US	S
U0S9  [        (       a  [        R                  X5      nO
[        SSS9eSU;   a  US   SL a  [        U5      $ UR                  US	US
U0S9  U" SU 35        US   R                  S   R                  UR                  S   R                  l        [!        US   R"                  5      n[!        US   R                  S   R$                  5      n['        [(        R(                  " 5       5      Ul        Xl        [/        UUUU-   S9n[1        USU5        U$ ! [         a  n[        S[	        U5      S9eS nAff = f)Nr'   r   r(   rolesinitial_prompt_valuefinal_prompt_value	role_dictr5   r6   r/   r#   r/    complete_input_dict)inputapi_keyadditional_args/Need to pass in a model name to initialize vllmstreamT)r<   r=   original_responser>   zraw model_response: prompt_tokenscompletion_tokenstotal_tokensusage )r.   r*   r+   r	   r,   r   r   pre_callgenerateiter	post_calloutputstextchoicesr   contentlenprompt_token_ids	token_idsinttimecreatedr#   r   setattr)r#   r/   r0   r1   encodinglogging_objr2   custom_prompt_dictlitellm_params	logger_fnr&   r-   sampling_paramsmodel_prompt_detailspromptrL   rC   rD   rF   s                      r   
completionr_   '   s   72?^ %77O"18*73!56L!M34HI	
  e? .@   s,,v7#T
 	

 ?"x'@D'HG} 	%2OD	 	 	
 	,WI674;AJ4F4Fq4I4N4Nq!))1 GAJ778
 2 21 5 ? ?@!$TYY[!1$'/&)::

 	/q  7As1v667s   E/ /
F9FFc                 ,    [        U S9u  pEU" S0 UD6n/ n	X;   a4  X0   n
U H)  n[	        U
S   U
S   U
S   US9nU	R                  U5        M+     O#U H  n[        XS	9nU	R                  U5        M     U(       a  UR                  X5      nO
[        SS
S9e/ nU H  n[        5       nUR                  S   R                  UR                  S   R                  l        [        UR                  5      n[        UR                  S   R                   5      n[#        [$        R$                  " 5       5      Ul        U Ul        [+        UUUU-   S9n[-        USU5        UR                  U5        M     U$ ! [         a  n[        U5      n[        SUS9eSnAff = f)a  
Example usage:
import litellm
import os
from litellm import batch_completion


responses = batch_completion(
    model="vllm/facebook/opt-125m",
    messages = [
        [
            {
                "role": "user",
                "content": "good morning? "
            }
        ],
        [
            {
                "role": "user",
                "content": "what's the time? "
            }
        ]
    ]
)
r'   r   r(   Nr4   r5   r6   r7   r9   r?   rB   rF   rG   )r.   r+   r,   r	   r   appendr   rI   r   rL   rM   rN   r   rO   rP   rQ   rR   rS   rT   rU   r#   r   rV   )r#   r/   r2   rY   r*   r&   r-   	error_strr\   promptsr]   r   r^   rL   final_outputsoutputr0   rC   rD   rF   s                       r   batch_completionsrf   q   s   8:2? %77OG"18G".w7%9:P%Q#78L#M 	F NN6"    G#%BFNN6"   ,,w8#T
 	
 M&4:NN14E4J4Jq!))1 F334q 1 ; ;<!$TYY[!1$'/&)::

 	/^,# $ _  :F	Ay99:s   E/ /
F9FFc                      g )NrG   rG   r   r   	embeddingrh      s    r   )rT   typingr   r   3litellm.litellm_core_utils.prompt_templates.factoryr   r   litellm.utilsr   r   r*   r+   r	   r,   r.   listdictr_   rf   rh   rG   r   r   <module>rn      s       /

	 
	7 	7( GGG "G 	G GV 15MMM`	r   