
    ht              
          S SK r S SKrS SKrS SKrS SKJr  S SKJr  S SKJ	r	J
r
  S SKJr  S SKJr  S SKJrJr  S SKJr  S S	KJr  S SKrS S
KJr  S SKJr  S SKJrJrJrJrJ r   \" 5       (       a  S SK!J"r"  \" 5       (       a  S SK#J$r$  \" 5       (       a  S SK%r%\" 5       (       a  S SK&J'r'  \ " 5       (       a8  S SK(J)r)J*r*  S SK+J,r,  S SK-J.r.  S SK/J0r0  S SK1J2r2  S SK3J4r4  \" 5       (       a  S SK5J6r,  \Rn                  " \85      r9S\Rt                  S'    " S S5      r;\	 " S S5      5       r<S\<S\=S\=S \S!S4
S" jr>S#\?S$\=S!\?\?   4S% jr@S& rAS\<4S' jrBS+S(\ R                  4S) jjrD\8S*:X  a"  \D" 5       rE\ER                  5       u  rG\B" \G5        gg),    N)Sequence)asynccontextmanager)	dataclassfield)BytesIO)chain)PipeProcess)
Connection)Optional)is_vision_available)	TrlParser)is_fastapi_availableis_pydantic_availableis_uvicorn_availableis_vllm_ascend_availableis_vllm_available)FastAPI)	BaseModel)Image)LLMSamplingParams)PyNcclCommunicator)get_world_group)StatelessProcessGroup)GuidedDecodingParams)get_open_port)PyHcclCommunicatorspawnVLLM_WORKER_MULTIPROC_METHODc            
       h    \ rS rSrSrSrSrS\S\S\S\SS4
S	 jr	S
\S\S\
\   SS4S jrSS jrSrg)WeightSyncWorkerExtensionM   a  
A vLLM worker extension that enables weight synchronization between a client and multiple server workers.

This worker uses a `StatelessProcessGroup` to establish communication and a `PyNcclCommunicator` to handle
efficient GPU-based communication using NCCL. The primary purpose of this class is to receive updated model weights
from a client process and distribute them to all worker processes participating in model inference.
Nhostport
world_sizeclient_device_uuidreturnc                 h   U R                   b  [        S5      eU[        [        R                  R                  U R                  5      R                  5      :X  a  [        SU S35      e[        5       R                  n[        R                  " XXSS9n[        X`R                  S9U l         US-
  U l        g)a  
Initializes the weight update communicator using a stateless process group.

This method creates a `StatelessProcessGroup` that allows external training processes to communicate with vLLM
workers without interfering with the global torch distributed group.

Args:
    host (`str`):
        Hostname or IP address of the master node.
    port (`int`):
        Port number to be used for communication.
    world_size (`int`):
        Total number of participating processes in the update group.
    client_device_uuid (`str`):
        UUID of the device of client main process. Used to assert that devices are different from vllm workers
        devices.
NzGWeight update group already initialized. Call close_communicator first.z.Attempting to use the same CUDA device (UUID: z) for multiple distinct roles/ranks within the same communicator. This setup is unsupported and will likely lead to program hangs or incorrect behavior. Ensure that trainer is using different devices than vLLM server.)r$   r%   rankr&   )device   )pynccl_commRuntimeErrorstrtorchcudaget_device_propertiesr+   uuidr   r*   r   creater   client_rank)selfr$   r%   r&   r'   r*   pgs          P/home/james-whalen/.local/lib/python3.13/site-packages/trl/scripts/vllm_serve.pyinit_communicator+WeightSyncWorkerExtension.init_communicatorZ   s    $ 'hiiUZZ%E%Edkk%R%W%W!XX@AS@T Up p   %% #))tTa .bE &>    namedtypeshapec                    U R                   c  [        S5      e[        [        UR	                  S5      S   5      n[        R
                  " X2U R                  S9nU R                   R                  X@R                  S9  U R                   R                  R                  5         U R                  R                  R                  X4/S9  g)aL  
Receives updated weights from the client process and updates the named parameter in the model.

Args:
    name (`str`):
        Name of the weight tensor being updated.
    dtype (`str`):
        Data type of the weight tensor as a string (e.g., `"torch.float32"`).
    shape (`Sequence[int]`):
        Shape of the weight tensor.
Nz=Communicator not initialized. Call `init_communicator` first..)r=   r+   )src)weights)r-   r.   getattrr0   splitemptyr+   	broadcastr5   groupbarriermodel_runnermodelload_weights)r6   r<   r=   r>   weights        r8   update_named_param,WeightSyncWorkerExtension.update_named_param   s     #^__u{{3/34UD 	""6/?/?"@&&( 	,,tn5E,Fr;   c                 @    U R                   b  U ? SU l         SU l        gg)z
Closes the communicator when weight synchronization is no longer needed.

This method deletes the NCCL communicator to release associated resources.
N)r-   r5   )r6   s    r8   close_communicator,WeightSyncWorkerExtension.close_communicator   s+     ' #D#D (r;   )r5   r-   )r(   N)__name__
__module____qualname____firstlineno____doc__r-   r5   r/   intr9   r   rN   rQ   __static_attributes__ r;   r8   r"   r"   M   sn     KK%*c %* %*# %*[^ %*cg %*NGs G3 Gx} GQU G4
$r;   r"   c                      \ rS rSr% Sr\" SS0S9r\\S'   \" SSS0S	9r	\
\   \S
'   \" SSS0S	9r\\S'   \" SSS0S	9r\\S'   \" SSS0S	9r\\S'   \" SSS0S	9r\\S'   \" SSS0S	9r\\S'   \" SSS0S	9r\\S'   \" SSS0S	9r\
\   \S'   \" SSS0S	9r\
\   \S'   \" S SS!0S	9r\
\   \S"'   \" SSS#0S	9r\\S$'   \" S SS%0S	9r\\S&'   \" S'SS(0S	9r\\S)'   \" S*SS+0S	9r\\S,'   S-rg).ScriptArguments   aW  
Arguments for the script.

Args:
    model (`str`):
        Model name or path to load the model from.
    revision (`str` or `None`, *optional*, defaults to `None`):
        Revision to use for the model. If not specified, the default branch will be used.
    tensor_parallel_size (`int`, *optional*, defaults to `1`):
        Number of tensor parallel workers to use.
    data_parallel_size (`int`, *optional*, defaults to `1`):
        Number of data parallel workers to use.
    host (`str`, *optional*, defaults to `"0.0.0.0"`):
        Host address to run the server on.
    port (`int`, *optional*, defaults to `8000`):
        Port to run the server on.
    gpu_memory_utilization (`float`, *optional*, defaults to `0.9`):
        Ratio (between 0 and 1) of GPU memory to reserve for the model weights, activations, and KV cache on the
        device dedicated to generation powered by vLLM. Higher values will increase the KV cache size and thus
        improve the model's throughput. However, if the value is too high, it may cause out-of-memory (OOM) errors
        during initialization.
    dtype (`str`, *optional*, defaults to `"auto"`):
        Data type to use for vLLM generation. If set to `"auto"`, the data type will be automatically determined
        based on the model configuration. Find the supported values in the vLLM documentation.
    max_model_len (`int` or `None`, *optional*, defaults to `None`):
        If set, the `max_model_len` to use for vLLM. This can be useful when running with reduced
        `vllm_gpu_memory_utilization`, leading to a reduced KV cache size. If not set, vLLM will use the model
        context size, which might be much larger than the KV cache, leading to inefficiencies.
    enable_prefix_caching (`bool` or `None`, *optional*, defaults to `None`):
        Whether to enable prefix caching in vLLM. If set to `True`, ensure that the model and the hardware support
        this feature.
    enforce_eager (`bool`, *optional*, defaults to `False`):
        Whether to enforce eager execution. If set to `True`, we will disable CUDA graph and always execute the
        model in eager mode. If `False` (default behavior), we will use CUDA graph and eager execution in hybrid.
    vllm_model_impl (`str`, *optional*, defaults to `"vllm"`):
        Model implementation to use for vLLM. Must be one of `"transformers"` or `"vllm"`. `"transformers"`: Use
        the `transformers` backend for model implementation. `"vllm"`: Use the `vllm` library for model
        implementation.
    kv_cache_dtype (`str`, *optional*, defaults to `"auto"`):
        Data type to use for KV cache. If set to `"auto"`, the dtype will default to the model data type.
    trust_remote_code (`bool`, *optional*, defaults to `False`):
        Whether to trust remote code when loading models. Set to `True` to allow executing code from model
        repositories. This is required for some custom models but introduces security risks.
    log_level (`str`, *optional*, defaults to `"info"`):
        Log level for uvicorn. Possible choices: `"critical"`, `"error"`, `"warning"`, `"info"`, `"debug"`,
        `"trace"`.
helpz*Model name or path to load the model from.)metadatarK   NzQRevision to use for the model. If not specified, the default branch will be used.)defaultr_   revisionr,   z)Number of tensor parallel workers to use.tensor_parallel_sizez'Number of data parallel workers to use.data_parallel_sizez0.0.0.0z"Host address to run the server on.r$   i@  zPort to run the server on.r%   g?aQ  Ratio (between 0 and 1) of GPU memory to reserve for the model weights, activations, and KV cache on the device dedicated to generation powered by vLLM. Higher values will increase the KV cache size and thus improve the model's throughput. However, if the value is too high, it may cause out-of-memory (OOM) errors during initialization.gpu_memory_utilizationautozData type to use for vLLM generation. If set to 'auto', the data type will be automatically determined based on the model configuration. Find the supported values in the vLLM documentation.r=   a  If set, the `max_model_len` to use for vLLM. This can be useful when running with reduced `vllm_gpu_memory_utilization`, leading to a reduced KV cache size. If not set, vLLM will use the model context size, which might be much larger than the KV cache, leading to inefficiencies.max_model_lenzxWhether to enable prefix caching in vLLM. If set to `True`, ensure that the model and the hardware support this feature.enable_prefix_cachingFzWhether to enforce eager execution. If set to `True`, we will disable CUDA graph and always execute the model in eager mode. If `False` (default behavior), we will use CUDA graph and eager execution in hybrid.enforce_eagerz_Data type to use for KV cache. If set to 'auto', the dtype will default to the model data type.kv_cache_dtypezWhether to trust remote code when loading models. Set to True to allow executing code from model repositories. This is required for some custom models but introduces security risks.trust_remote_codeinfozbLog level for uvicorn. Possible choices: 'critical', 'error', 'warning', 'info', 'debug', 'trace'.	log_levelvllmzModel implementation to use for vLLM. Must be one of `transformers` or `vllm`. `transformers`: Use the `transformers` backend for model implementation. `vllm`: Use the `vllm` library for model implementation.vllm_model_implrZ   )rS   rT   rU   rV   rW   r   rK   r/   __annotations__ra   r   rb   rX   rc   r$   r%   rd   floatr=   rf   rg   boolrh   ri   rj   rl   rn   rY   rZ   r;   r8   r\   r\      s   .` FGE3  $mnHhsm  !&EF!#  $CD  >?D#  67D#  %* @
%E   p
E3  $) e
$M8C=  -2 -
-8D>  %* #
%M8D>   u
NC  $ c
t   
Is  ! $
OS r;   r\   script_argsdata_parallel_rankmaster_port
connectionr(   c                 X   [        U5      [        R                  S'   [        U5      [        R                  S'   [        U R                  5      [        R                  S'   [        U5      [        R                  S'   [	        U R
                  U R                  U R                  U R                  U R                  U R                  U R                  U R                  U R                  SU R                  U R                  S9nUR!                  SS05          UR#                  5       nUS   S;   aV  US   nUR)                  SS5      UR)                  S0 5      p[+        XF5      n	U	" U0 UD6n
US   S:X  a  UR!                  U
5        O
US   S:X  a  g M|  ! [$         a    UR'                  S	S
9   g f = f)NVLLM_DP_RANKVLLM_DP_RANK_LOCALVLLM_DP_SIZEVLLM_DP_MASTER_PORTz0trl.scripts.vllm_serve.WeightSyncWorkerExtension)rK   ra   rb   rd   rh   r=   rg   ri   rf   worker_extension_clsrj   
model_implstatusreadyrQ   )methodtype)callfire_and_forgetr   argsrZ   kwargsr   shutdown)r/   osenvironrc   r   rK   ra   rb   rd   rh   r=   rg   ri   rf   rj   rn   sendrecvKeyboardInterruptcollective_rpcgetrD   )rr   rs   rt   ru   llmcommandmethod_namer   r   r   results              r8   
llm_workerr   6  s    "%%7!8BJJ~'*+='>BJJ#$!$[%C%C!DBJJ~(+K(8BJJ$%
%%(==*AA!// *??"11!//O%77..C& OOXw'(
	 oo'G 6?99!(+K";;vr2GKK"4M&S.FT,V,Fv&('V_
*#  ! 	&:;	s   F F)(F)lstnc           
          [        [        U 5      U5      u  p#[        U5       Vs/ s H+  o@XB-  [        XC5      -   US-   U-  [        US-   U5      -    PM-     sn$ s  snf )a  
Split list `lst` into `n` evenly distributed sublists.

Example:
```python
>>> chunk_list([1, 2, 3, 4, 5, 6], 2)
[[1, 2, 3], [4, 5, 6]]

>>> chunk_list([1, 2, 3, 4, 5, 6], 4)
[[1, 2], [3, 4], [5], [6]]

>>> chunk_list([1, 2, 3, 4, 5, 6], 8)
[[1], [2], [3], [4], [5], [6], [], []]
```
r,   )divmodlenrangemin)r   r   kris        r8   
chunk_listr   h  sZ      #c(ADAJOPQ(S(QA	!QUaK#a!eQ-$?@(SSSs   2Ac                     SS K nU R                  nUR                  U5      (       a  [        R	                  SU  S35        g U$ )Nr   z&Generated NaN logprob, token logprob 'z' will be ignored)mathlogprobisnanloggerwarning)r   r   values      r8   sanitize_logprobr   |  s<    OOEzz%?yHYZ[Lr;   c                 r  ^ ^^ [        5       (       d  [        S5      e[        5       (       d  [        S5      e[        5       (       d  [        S5      e[	        5       (       d  [        S5      e[        5       n/ m/ m[        T R                  5       HR  n[        5       u  p4[        [        T X!U4S9nUR                  5         TR                  U5        TR                  U5        MT     [        S[        4UUU 4S jj5       n[        US9nUR                  S	5      S
 5       nUR                  S5      U 4S j5       n	 " S S[         5      n
 " S S[         5      nUR#                  SUS9SU
4UU 4S jj5       n " S S[         5      nUR#                  S5      SU4UU 4S jj5       n " S S[         5      nUR#                  S5      SU4U4S jj5       nUR#                  S5      U4S j5       nUR#                  S5      U4S  j5       n[$        R&                  " UT R(                  T R*                  T R,                  S!9  g )"Nz`FastAPI is required to run the vLLM serve script. Please install it using `pip install fastapi`.zbPydantic is required to run the vLLM serve script. Please install it using `pip install pydantic`.z`Uvicorn is required to run the vLLM serve script. Please install it using `pip install uvicorn`.zZvLLM is required to run the vLLM serve script. Please install it using `pip install vllm`.)targetr   appc                  >#    [        5       n[        U5      TR                  :  as  T HR  nUR                  5       n[	        U[
        5      (       d  M*  UR                  S5      S:X  d  MA  UR                  U5        MT     [        U5      TR                  :  a  Ms  S 7v   T Hb  nUR                  SS9  UR                  5       (       d  M)  [        R                  SU S35        UR                  5         UR                  5         Md     g 7f)Nr}   r~   
   )timeoutzProcess z< is still alive after 10 seconds, attempting to terminate...)setr   rc   r   
isinstancedictr   addjoinis_aliver   r   	terminate)r   ready_connectionsru   msgprocessconnections	processesrr   s        r8   lifespanmain.<locals>.lifespan  s       E#${'E'EE)
 oo'c4((SWWX->'-I%))*5 * #${'E'EE 	 !GLLL$!!'2nop!!# !s   ADD*.D,D
>D)r   z/health/c                     #    SS0$ 7f)z=
Health check endpoint to verify that the server is running.
r}   okrZ   rZ   r;   r8   healthmain.<locals>.health  s     
 $s   z/get_world_size/c                  B   >#    ST R                   T R                  -  0$ 7f)z
Retrieves the world size of the LLM engine, which is `tensor_parallel_size * data_parallel_size`.

Returns:
    `dict`:
        A dictionary containing the world size.

Example response:
```json
{"world_size": 8}
```
r&   )rb   rc   )rr   s   r8   get_world_sizemain.<locals>.get_world_size  s$      k>>A_A__``s   c                       \ rS rSr% \\   \S'   Sr\\\      \S'   Sr	\
\S'   Sr\\S'   Sr\\S	'   Sr\\S
'   Sr\
\S'   Sr\\S'   Sr\
\S'   Sr\\   \S'   \" \S9r\\S'   Srg)main.<locals>.GenerateRequesti  promptsNimagesr,   r   g      ?repetition_penaltytemperaturetop_prA   top_kg        min_p   
max_tokensguided_decoding_regex)default_factorygeneration_kwargsrZ   )rS   rT   rU   rV   listr/   ro   r   r   r   rX   r   rp   r   r   r   r   r   r   r   r   r   rY   rZ   r;   r8   GenerateRequestr     s    c&*c#*3
$'E' U usu
C/3x}3"'"=4=r;   r   c                   B    \ rS rSr% \\\      \S'   \\\      \S'   Srg)main.<locals>.GenerateResponsei  completion_idslogprobsrZ   N)	rS   rT   rU   rV   r   rX   ro   rp   rY   rZ   r;   r8   GenerateResponser     s    T#Y'tE{##r;   r   z
/generate/)response_modelrequestc                   >#    U R                   =(       d    S/[        U R                  5      -  U l         / n[        U R                  U R                   5       HU  u  p#SU0nUb8  S[        R
                  " [        [        R                  " U5      5      5      0US'   UR                  U5        MW     U R                  b  [        SU R                  S9nOSnU R                  U R                  U R                  U R                  U R                   U R"                  U R$                  USS.	nUR'                  U R(                  5        [+        S0 UD6n[-        UTR.                  5      n[        TU5       H(  u  pU(       d  S	/nXS
.n
U	R1                  SSU
S.5        M*     T V	s/ s H  oR3                  5       PM     nn	[        X5       VVs/ s H  u  pU(       d  M  UPM     nnn[5        [6        R8                  " U5      5      nU VVs/ s H+  oR:                    H  n[5        UR<                  5      PM     M-     nnnU VVVs/ s H]  nUR:                    HI  nUR>                   Vs/ s H.  n[A        [C        [E        URG                  5       5      5      5      PM0     snPMK     M_     nnnnUUS.$ s  sn	f s  snnf s  snnf s  snf s  snnnf 7f)a  
Generates completions for the provided prompts.

Args:
    request (`GenerateRequest`):
        - `prompts` (list of `str`): A list of prompts (text strings) for the model to generate completions.
        - `images` (list of `str`, *optional*, default to `None`): A list of base64 encoded images to process
          along with prompts.
        - `n` (`int`, *optional*, defaults to `1`): Number of completions to generate for each prompt.
        - `repetition_penalty` (`float`, *optional*, defaults to `1.0`): Repetition penalty to apply during
          generation.
        - `temperature` (`float`, *optional*, defaults to `1.0`): Temperature for sampling. Higher values lead
          to more random outputs.
        - `top_p` (`float`, *optional*, defaults to `1.0`): Top-p (nucleus) sampling parameter. It controls the
          diversity of the generated text.
        - `top_k` (`int`, *optional*, defaults to `-1`): Top-k sampling parameter. If set to `-1`, it disables
          top-k sampling.
        - `min_p` (`float`, *optional*, defaults to `0.0`): Minimum probability threshold for sampling.
        - `max_tokens` (`int`, *optional*, defaults to `16`): Maximum number of tokens to generate for each
          completion.
        - `guided_decoding_regex` (`str`, *optional*): A regex pattern for guided decoding. If provided, the
          model will only generate tokens that match this regex pattern.
        - `generation_kwargs` (`dict`, *optional*): Additional generation parameters to pass to the vLLM
          `SamplingParams`. This can include parameters like `seed`, `frequency_penalty`, etc. If it contains
          keys that conflict with the other parameters, they will override them.

Returns:
    `GenerateResponse`:
        - `completion_ids` (list of list of `int`): A list of lists of token IDs for each generated completion.
        - `logprobs` (list of list of `float`): A list of lists of log probabilities for each token in the
          generated completions.

Example request:
```json
{"prompts": ["Hello world", "What is AI?"]}
```

Example response:
```json
{"completion_ids": [[101, 102, 103], [201, 202, 203]], "logprobs": [[-0.1, -0.2, -0.3], [-0.4, -0.5, -0.6]]}
```
Npromptimagemulti_modal_dataoutlines)backendregexr   )	r   r   r   r   r   r   r   guided_decodingr   z<placeholder>)r   sampling_paramsr   generater   r   r   )r   r   rZ   )$r   r   r   zipr   openr   base64	b64decodeappendr   r   r   r   r   r   r   r   r   updater   r   r   rc   r   r   r   r   from_iterableoutputs	token_idsr   r   nextitervalues)r   r   r   r   rowr   r   r   chunked_promptsru   r   all_outputsoutputr   r   r   r   r   rr   s                    r8   r   main.<locals>.generate  s}    X !HD6C4H+H '..AMFV$C +2EJJwvGWGWX]G^?_4`*a&'NN3	 B ((42:WMjMjkO"O ")"<"<"..]]]]]]!,,.

 	  !:!:;(=+<= %Wk.L.LM $'{O#DJ *+!(MFOOVzVTU $E <GG;Z(;G 695Vb5V/&Zav5Vb 5..{;<@KjWZiZiPV$v//0Zi0j ''
&!// LR??[?d4(8#9:;?[) \& 	 '

 #1hGG H c k['
sT   FK
J-7K
J2J2'K
2J87	K
 $K$5J>K$K
>KK
c                   >    \ rS rSr% \\S'   \\S'   \\S'   \\S'   Srg)%main.<locals>.InitCommunicatorRequestiJ  r$   r%   r&   r'   rZ   N)rS   rT   rU   rV   r/   ro   rX   rY   rZ   r;   r8   InitCommunicatorRequestr   J  s    		r;   r   z/init_communicator/c                    >#    TR                   TR                  -  S-   nSU R                  U R                  XR                  4S.nT H  nUR                  SSUS.5        M     SS0$ 7f)	a  
Initializes the communicator for synchronizing model weights between a client and multiple server workers.

Args:
    request (`InitCommunicatorRequest`):
        - `host` (`str`): Hostname or IP address of the master node.
        - `port` (`int`): Port number to be used for communication.
        - `world_size` (`int`): Total number of participating processes in the group.
        - `client_device_uuid` (`str`): UUID of the device of client main process. Used to assert that devices
          are different from vLLM workers devices.
r,   r9   r   r   r   r   r   messagez+Request received, initializing communicator)rb   rc   r$   r%   r'   r   )r   r&   r   ru   r   rr   s       r8   r9   main.<locals>.init_communicatorP  sx      !558V8VVYZZ
 *\\7<<=W=WX
 &JOO%6BR^def & HIIs   A%A(c                   :    \ rS rSr% \\S'   \\S'   \\   \S'   Srg)"main.<locals>.UpdateWeightsRequestik  r<   r=   r>   rZ   N)	rS   rT   rU   rV   r/   ro   r   rX   rY   rZ   r;   r8   UpdateWeightsRequestr   k  s    	
Cyr;   r   z/update_named_param/c                    >#    SU R                   U R                  [        U R                  5      4S.nT H  nUR	                  SSUS.5        M     SS0$ 7f)a  
Updates the model weights with the provided tensor.

Once this endpoint is called, the client process should broadcast the updated weights to all server workers.

Args:
    request (`UpdateWeightsRequest`):
        - `name` (`str`): Name of the weight tensor being updated.
        - `dtype` (`str`): Data type of the weight tensor (e.g., `"torch.float32"`).
        - `shape` (list of `int`): Shape of the weight

rN   r   r   r   r   r   z*Request received, updating named parameter)r<   r=   tupler>   r   )r   r   ru   r   s      r8   rN    main.<locals>.update_named_paramp  s\     " 17<<X]^e^k^kXl:mn%JOO%6BR^def & GHHs   AAz/reset_prefix_cache/c                     >#    T H  n U R                  SSS.5        M     T V s/ s H  o R                  5       PM     nn [        S U 5       5      nSS[        U5      -   0$ s  sn f 7f)z(
Resets the prefix cache for the model.
r   reset_prefix_cache)r   r   c              3   $   #    U  H  ov   M     g 7fNrZ   ).0r   s     r8   	<genexpr>3main.<locals>.reset_prefix_cache.<locals>.<genexpr>  s     7;f;s   r   z1Request received, resetting prefix cache status: )r   r   allr/   )ru   r   successr   s      r8   r   main.<locals>.reset_prefix_cache  sn     
 &JOOV7KLM & <GG;Z(;G7;77NQTU\Q]]^^ Hs   "A'A")A'z/close_communicator/c                  X   >#    SS0n T H  nUR                  SSU S.5        M     SS0$ 7f)zD
Closes the weight update group and cleans up associated resources.
r   rQ   r   r   r   r   z&Request received, closing communicator)r   )r   ru   r   s     r8   rQ    main.<locals>.close_communicator  s?     
 01%JOO%6BR^def &CDDs   '*)r$   r%   rl   )r   ImportErrorr   r   r   r   r   rc   r	   r
   r   startr   r   r   r   r   postuvicornrunr$   r%   rl   )rr   rt   rs   parent_connectionchild_connectionr   r   r   r   r   r   r   r   r   r9   r   rN   r  rQ   r   r   s   `                  @@r8   mainr    sZ   !!n
 	
 !""p
 	
  !!n
 	
 vww  /KKI#K$B$BC.2f+;@Raq2rs,-! D G   & 8
$C 	WWZ    	WW a !a>) >$9 $ 	XXl+;X<cH cH =cHJ )   	XX#$J)@ J %J4y 
 	XX$%I*> I &I, 	XX$%	_ &	_ 	XX$%E &E KK+**1A1A[MbMbcr;   
subparsersc                 X    U b  U R                  SS[        S9nU$ [        [        5      nU$ )Nz
vllm-servezRun the vLLM serve script)r^   dataclass_types)
add_parserr\   r   )r  parsers     r8   make_parserr    s9    &&|:Ugv&w M ?+Mr;   __main__r  )Hargparser   loggingr   collections.abcr   
contextlibr   dataclassesr   r   ior   	itertoolsr   multiprocessingr	   r
   multiprocessing.connectionr   typingr   r0   transformersr   trlr   trl.import_utilsr   r   r   r   r   fastapir   pydanticr   r  PILr   rm   r   r   ,vllm.distributed.device_communicators.pyncclr   vllm.distributed.parallel_stater   vllm.distributed.utilsr   vllm.sampling_paramsr   
vllm.utilsr   3vllm_ascend.distributed.device_communicators.pyhcclr   	getLoggerrS   r   r   r"   r\   rX   r   r   r   r   r  _SubParsersActionr  r  parse_args_and_configrr   rZ   r;   r8   <module>r4     s      	 $ * (   ) 1   ,    "   (O?<9(!!p 
		8	$
 .5

) *X$ X$v J J JZ/ /69/HK/Yc/	/dTD TS TT$Z T(Wdo WdtH66  z]F113N[ r;   