
    :iJ              
          S SK r S SKJrJrJr  S SKJrJrJr  S SK	J
r
JrJrJrJrJrJrJrJrJrJr  S SKJrJr  S SKJrJrJr  S SKJrJr  Sr " S	 S
\\\\\\4   5      r  " S S\ 5      r! " S S\!5      r"S\S\ 4S jr#S\S\ 4S jr$g)    N)GenericSequenceoverload)	TextChunk
ThinkChunkUserContentChunk)UATSAssistantMessageAssistantMessageTypeFinetuningAssistantMessageRolesSystemMessageSystemMessageTypeToolMessageToolMessageTypeUserMessageUserMessageType)ChatCompletionRequestInstructRequest)FunctionCallToolToolCall)InstructRequestTypeTokenizerVersion

c                   r   \ rS rSr% SrSr\\S'   Sr\\S'   S\	\
   S\	\   S\	\   S	\	\   S
\	\   4
S jr\S$S j5       rS\S-  S\4S jr\S\\\-  \-     S\\\\-     -  4S j5       r\S\S\4S j5       r\S\\   S\4S j5       rS\\\\-  \-     -  \\   -  S\\\\-     -  4S jrS\\   S\S-  4S jrS\\   S\\   S\\   4S jrS\S\4S jrS\\   S\\   4S jrS\\   S\4S jrS\\   S\
4S jr S\\   S\!S-  S\\   S\"\   4S jr#S\\   S\\   4S  jr$S!\%\   S\4S" jr&S#r'g)%InstructRequestNormalizer   a  Takes a [ChatCompletionRequest][mistral_common.protocol.instruct.request.ChatCompletionRequest] and normalizes
it into an [InstructRequest][mistral_common.tokens.instruct.request.InstructRequest].

The normalization process does several things such as:
- Aggregate consecutive messages of the same role
- Aggregate system prompts
- Normalize json content
- Normalize tool calls

Examples:
    >>> normalizer = InstructRequestNormalizer.normalizer()
F_system_prompt_in_begin_allow_tool_call_and_contentuser_message_classassistant_message_classtool_message_classsystem_message_classinstruct_request_classc                 @    Xl         X l        X0l        XPl        X@l        g)ab  Initializes the normalizer with the appropriate message classes.

Args:
   user_message_class: The class for user messages.
   assistant_message_class: The class for assistant messages.
   tool_message_class: The class for tool messages.
   system_message_class: The class for system messages.
   instruct_request_class: The class for instruct requests.
N)_user_message_class_assistant_message_class_tool_message_class_instruct_request_class_system_message_class)selfr!   r"   r#   r$   r%   s         d/home/james-whalen/.local/lib/python3.13/site-packages/mistral_common/protocol/instruct/normalize.py__init__"InstructRequestNormalizer.__init__0   s"    " $6 (?%#5 '=$%9"    returnc            	      b    [        [        [        [        [        [
        [        [        4   5      $ )z~Returns a normalizer for the default instruct request.

Examples:
    >>> normalizer = InstructRequestNormalizer.normalizer()
)r   r   r
   r   r   r   r	   r    r0   r-   
normalizer$InstructRequestNormalizer.normalizerH   s)     )D$J'
 	
r0   contentNc                     Ub  [        U5      S:X  a  g [        R                  " U5      n[        R                  " USS9nU$ ! [        R                   a    Un U$ f = f)Nr   z{}F)ensure_ascii)lenjsonloadsdumpsJSONDecodeError)r,   r6   parsed_jsonnormalized_contents       r-   _normalize_json_content1InstructRequestNormalizer._normalize_json_contentW   sd    ?c'la/	)**W-K!%Ke!L "! ## 	)!(!!	)s   +A AAc                     g Nr3   r,   r6   s     r-   _aggregate_content_chunks3InstructRequestNormalizer._aggregate_content_chunksb   s     .1r0   c                     g rC   r3   rD   s     r-   rE   rF   f   s    >Ar0   c                     g rC   r3   rD   s     r-   rE   rF   h   s    DGr0   c                    [        U[        5      (       a  U$ [        U[        5      (       d   S[        U5       35       e/ nU H  n[        U[        5      (       a	  [	        US9n[        U[        5      (       a]  U(       aC  [        US   [        5      (       a+  US   =R
                  [        UR
                  -   -  sl        M  UR                  U5        M  [        U[        5      (       a  UR                  U5        M  [        S[        U5       35      e   [        U5      S:X  a'  [        US   [        5      (       a  US   R
                  $ U$ )NzExpected list, got textzUnsupported chunk type    r   )
isinstancestrlisttyper   rK   CHUNK_JOIN_STRappendr   
ValueErrorr9   )r,   r6   aggregated_contentchunks       r-   rE   rF   j   s    gs##N'4((O,?W*OO(;=E%%%!u-%++%*5G5KY*W*W&r*//>EJJ3NN/&--e4E:.."))%0 #:4;-!HII  !"a'J7I!7Li,X,X%a(---!!r0   messagesc                    / nU Hb  nUR                   [        R                  :X  d  M#  UR                  (       d  M6  U R	                  UR                  5      nUR                  U5        Md     [        U5      (       a  SR                  U5      $ S $ )Nr   )roler   systemr6   rE   rS   r9   join)r,   rW   system_promptmessagerU   s        r-   _aggregate_system_prompts3InstructRequestNormalizer._aggregate_system_prompts   sm    #%G||u||+%)%C%CGOO%T"$$%78  
 .1-?-?v{{=)ITIr0   latest_call_idsc           	         / nU H  n[        X@R                  5      (       d   S5       eU R                  UR                  5      nU R	                  U5      nUR                  U R                  XdR                  UR                  S95        M     U$ )z]
We currently do not do any aggregation for tool messages, but we normalize the json content
zExpected tool message)r6   tool_call_idname)rN   r)   rE   r6   r@   rS   rb   rc   )r,   rW   r`   tool_messagesr]   r6   r?   s          r-   _aggregate_tool_messages2InstructRequestNormalizer._aggregate_tool_messages   s     02Gg'?'?@@YBYY@44W__EG!%!=!=g!F  ((.=Q=QX_XdXd ) 	   r0   	tool_callc                     U R                  UR                  R                  5      n[        [	        UR                  R
                  US9UR                  S9$ )N)rc   	arguments)functionid)r@   rj   ri   r   r   rc   rk   )r,   rg   normalized_function_arumentss      r-   _normalize_tool_call.InstructRequestNormalizer._normalize_tool_call   sJ    '+'C'CIDVDVD`D`'a$!y'9'9'>'>Jfg||
 	
r0   c                     / $ rC   r3   r,   rW   s     r-   _aggregate_system_messages4InstructRequestNormalizer._aggregate_system_messages   s    	r0   c                     / n/ nSnS nU GH/  n[        X`R                  5      (       d   S5       eU R                  (       d0  UR                  (       a  UR                  (       a  [        SU 35      eUR                  (       a5  UR                   H%  nU R                  U5      nUR                  U5        M'     UR                  =n	b)  UR                  [        U	[        5      (       a  U	/OU	5        XFR                  -  n[        U[        5      (       d  GM
  Ub  XVR                  :X  d   S5       eUR                  nGM2     U(       a  U R                  U5      n
OS n
U R                  U
U=(       d    S US9nUb  [        US5      (       a  X[l        U$ )NFzExpected assistant messagezDTool calls and content cannot be used together in the same message. zEExpected weights of aggregated FinetuningAssistantMessage to be equal)r6   
tool_callsprefixweight)rN   r(   r    rt   r6   rT   rm   rS   extendrO   ru   r   rv   rE   hasattr)r,   rW   messages_contentsrt   ru   rv   r]   rg   normalized_tool_callr6   rU   aggregated_messages               r-   _aggregate_assistant_messages7InstructRequestNormalizer._aggregate_assistant_messages   s}   @B%'
#Gg'D'DEEcGccE44':L:LQXQ`Q` #ghogp!qrr!!!(!3!3I+/+D+DY+O(%%&:; "4 #??*7!((j#6N6N'T[\nn$F'#=>>%!^^3 _3 !-  0 !%!?!?@Q!R!%!::&!)T ; 
 '*<h"G"G(.%!!r0   c           	         / n/ nU H  n[        X@R                  5      (       d   S[        U5       35       e[        UR                  [        5      (       a  UR                  UR                  5        Ml  UR                   Hv  n[        U[        5      (       a  UR                  UR                  5        M5  U(       a)  UR                  [        SR                  U5      S95        / nUR                  U5        Mx     M     U(       a  SR                  U5      OSnU(       d  U R                  US9$ U(       a  UR                  [        US95        U R                  US9$ )z*
Just coalesce neighboring blocks of text
zExpected user message got r   rJ    r6   )	rN   r'   rQ   r6   rO   rS   r   rK   r[   )r,   rW   all_contenttext_chunksr]   rV   text_contents          r-   _aggregate_user_messages2InstructRequestNormalizer._aggregate_user_messages   s    /1!#Gg'?'?@@nD^_cdk_l^mBnn@'//3//""7??3$__E!%33#**5::6&'..yfkk+>V/WX*,K#**51 -   4?v{{;/B++L+AAyl;<'''<<r0   rY   c                    U[         R                  :X  a  U R                  X5      $ U[         R                  :X  a  U R	                  U5      /$ U[         R
                  :X  a  U R                  U5      /$ U R                  U5      $ rC   )r   toolre   	assistantr|   userr   rq   r,   rW   rY   r`   s       r-   _aggregate_role)InstructRequestNormalizer._aggregate_role   sp    5::00KKU__$66x@AAUZZ11(;<<228<<r0   c                 :   / n/ nS nS n/ nU GH  n[        USS 5      nXGR                  :w  d  X:w  a  UR                  U R                  X4U5      5        U[        R
                  :X  a[  US   n	[        US   [        5      (       d   eU	R                  b.  U	R                   H  n
UR                  U
R                  5        M      O$U[        R                  :X  a  UR                  5         UR                  5         UnUR                  nUR                  U5        GM     UR                  U R                  X4U5      5        [        U5      S:X  d2  U R                  (       d@  US   R                  [        R                  :w  a  UR!                  SU R#                  SS95        U$ )Nrv   rL   r   r   r   )getattrrY   rw   r   r   r   rN   r
   rt   rS   rk   r   clearr9   r   r   insertr'   )r,   rW   aggregated_messagesmessages_to_aggregatecurrent_rolecurrent_weightr`   r]   
new_weightassistant_messagerg   s              r-   _aggregate_messages-InstructRequestNormalizer._aggregate_messages  s|   *,,.%)'+%'  G (D9J||+
0L#**4+?+?@Uet+uv5??2(;B(?%%&9"&=?OPPPP(33?):)E)EI+229<<@ *F "UZZ/#))+%++-'N"<<L!((1'  , 	""4#7#78M]l#mn "#q(,,1DQ1G1L1LPUPZPZ1Z&&q$*B*B2*B*NO""r0   requestc                     U R                  UR                  5      nU R                  UR                  5      nU R                  UUUR                  UR
                  S9$ )aK  Converts a chat completion request to an instruct request.

Args:
    request: The chat completion request to convert.

Returns:
    The converted instruct request.

Examples:
    >>> from mistral_common.protocol.instruct.messages import UserMessage, AssistantMessage
    >>> request = ChatCompletionRequest(
    ...     messages=[
    ...         UserMessage(content="Hello"),
    ...         AssistantMessage(content="Hi"),
    ...     ],
    ... )
    >>> normalizer = InstructRequestNormalizer.normalizer()
    >>> instruct_request = normalizer.from_chat_completion_request(request)
)rW   r\   available_toolscontinue_final_message)r^   rW   r   r*   toolsr   )r,   r   r\   rW   s       r-   from_chat_completion_request6InstructRequestNormalizer.from_chat_completion_request+  s^    ( 66w7G7GH++G,<,<=++'#MM#*#A#A	 , 
 	
r0   )r(   r*   r+   r)   r'   )r1   r   )(__name__
__module____qualname____firstlineno____doc__r   bool__annotations__r    rQ   r   r   r   r   r   r.   staticmethodr4   rO   r@   r   rP   r   r   rE   r	   r^   re   r   rm   rq   r|   r   r   r   r   r   r   r   __static_attributes__r3   r0   r-   r   r      sh    %*T)). $.: 1: "&&:!;: !1	:
 ##45: !%%8 9:0 
 
	"sTz 	"c 	" 1C)Oj891	tI
*+	+1 1 AAA AGcGsG G"T#	/J">??$s)K"	tI
*+	+"8J$t* Jt Jd dSVi \`ap\q "
h 
8 
4: $GXBY +"d4j +"EY +"Z=d = =@=T
 =%$, =Y]^aYb =goptgu ='#DJ '#4: '#R
4I$4O 
Tg 
r0   r   c            	           \ rS rSr% SrSr\\S'   Sr\\S'   \	SS j5       r
S\\   S\\   4S	 jrS\\   S
\S-  S\\   S\\   4S jrS\\   S\S-  4S jrS\\   S\4S jrSrg)InstructRequestNormalizerV7iJ  zjNormalizer for the v7 tokenizer.

Examples:
    >>> normalizer = InstructRequestNormalizerV7.normalizer()
Tr   r    r1   c            	      b    [        [        [        [        [        [
        [        [        4   5      $ )zReturns a normalizer for the default instruct request

Examples:
    >>> normalizer = InstructRequestNormalizerV7.normalizer()
)r   r   r
   r   r   r   r	   r   r3   r0   r-   r4   &InstructRequestNormalizerV7.normalizerT  s)     +D$J'
 	
r0   rW   c                     U Vs/ s HG  n[        X R                  5      (       d  M  U R                  U R                  UR                  5      S9PMI     sn$ s  snf )Nr   )rN   r+   rE   r6   )r,   rW   r]   s      r-   rq   6InstructRequestNormalizerV7._aggregate_system_messagesc  sW     $
#'#=#=> `D&&t/M/Mgoo/^&_#
 	
 
s
   A,ArY   Nr`   c                 Z   U[         R                  :X  a  U R                  X5      $ U[         R                  :X  a  U R	                  U5      /$ U[         R
                  :X  a  U R                  U5      /$ U[         R                  :X  a  U R                  U5      $ Uc  [        U5      S:X  d   e/ $ )Nr   )
r   r   re   r   r|   r   r   rZ   rq   r9   r   s       r-   r   +InstructRequestNormalizerV7._aggregate_rolej  s    5::00KKU__$66x@AAUZZ11(;<<U\\!228<<<CMQ$666Ir0   c                     [        S5      e)Nz&We should not aggregate system prompts)NotImplementedErrorrp   s     r-   r^   5InstructRequestNormalizerV7._aggregate_system_promptsw  s    !"JKKr0   r   c                 n    U R                  UR                  5      nU R                  USUR                  S9$ )aM  Converts a chat completion request to an instruct request.

Args:
    request: The chat completion request to convert.

Returns:
    The converted instruct request.

Examples:
    >>> from mistral_common.protocol.instruct.messages import UserMessage, AssistantMessage
    >>> request = ChatCompletionRequest(
    ...     messages=[
    ...         UserMessage(content="Hello"),
    ...         AssistantMessage(content="Hi"),
    ...     ],
    ... )
    >>> normalizer = InstructRequestNormalizerV7.normalizer()
    >>> instruct_request = normalizer.from_chat_completion_request(request)
N)rW   r\   r   )r   rW   r*   r   )r,   r   rW   s      r-   r   8InstructRequestNormalizerV7.from_chat_completion_requestz  s9    ( ++G,<,<=++XTcjcpcp+qqr0   r3   )r1   r   )r   r   r   r   r   r   r   r   r    r   r4   rP   r	   r   rq   r   rO   r   r   r^   r   r   r   r   r3   r0   r-   r   r   J  s     %)T()- $-
 

4: 
$GXBY 
T
 %$, Y]^aYb goptgu L$t* Lt Lr4I$4O rTg rr0   r   c                   b   ^  \ rS rSrSr\S	S j5       rS\\   S\\	   S\\
   4U 4S jjrSrU =r$ )
InstructRequestNormalizerV13i  zNormalizer for the v13 tokenizer.

It reorders tool messages based on the tool call order.

Examples:
    >>> normalizer = InstructRequestNormalizerV13.normalizer()
r1   c            	      b    [        [        [        [        [        [
        [        [        4   5      $ )z6Returns a normalizer for the default instruct request.)r   r   r
   r   r   r   r	   r   r3   r0   r-   r4   'InstructRequestNormalizerV13.normalizer  s)     ,D$J'
 	
r0   rW   r`   c                    >^^ [         T	U ]  X5      n[        U5       VVs0 s H  u  pEXT_M	     snnm[        U5       VVs0 s H  u  pFUR                  U_M     snnmUR	                  UU4S jS9  U$ s  snnf s  snnf )Nc                 ~   > TR                  U R                  =(       d    S[        S5      5      TU R                     4$ )Nnullinf)getrb   float)msgid_to_tool_call_idxid_to_tool_result_idxs    r-   <lambda>GInstructRequestNormalizerV13._aggregate_tool_messages.<locals>.<lambda>  s5    #''(8(8(BFE%LQ%c&6&67r0   )key)superre   	enumeraterb   sort)
r,   rW   r`   rd   idxcall_idr]   r   r   	__class__s
          @@r-   re   5InstructRequestNormalizerV13._aggregate_tool_messages  s    /4w/OPX/j@I/@Z[@Zw|@Z[OXYfOg hOg|s!5!5s!:Og h 	 	
  \ hs   A4A:r3   )r1   r   )r   r   r   r   r   r   r4   rP   r	   rO   r   re   r   __classcell__)r   s   @r-   r   r     sK     
 
d dSVi \`ap\q  r0   r   versionr1   c                 l   U [         R                  [         R                  [         R                  1;   a  [        R                  5       $ U [         R                  [         R                  1;   a  [        R                  5       $ U [         R                  :X  a  [        R                  5       $ [        SU  35      e)a&  Gets the appropriate normalizer for the given tokenizer version.

Args:
    version: The tokenizer version to get the normalizer for.

Returns:
    The appropriate normalizer for the given tokenizer version.

Examples:
    >>> normalizer = normalizer_for_tokenizer_version(TokenizerVersion.v1)
zUnknown tokenizer version )r   v1v2v3r   r4   v7v11r   v13r   rT   )r   s    r-    normalizer_for_tokenizer_versionr     s     #&&(8(;(;=M=P=PQQ(3355	%((*:*>*>?	?*5577	$((	(+6688
1';
<<r0   c                    U [         R                  ::  a  [        nUR                  5       $ U [         R                  ::  a  [        nUR                  5       $ U [         R
                  ::  a  [        nUR                  5       $ [        SU  35      e)NzUnsupported tokenizer version: )	r   r   r   r   r   r   r   rT   r4   )r   normalizer_clss     r-   get_normalizerr     s    "%%%2 $$&& 
$''	'4 $$&& 
$((	(5 $$&& :7)DEEr0   )%r:   typingr   r   r   &mistral_common.protocol.instruct.chunkr   r   r   )mistral_common.protocol.instruct.messagesr	   r
   r   r   r   r   r   r   r   r   r   (mistral_common.protocol.instruct.requestr   r   +mistral_common.protocol.instruct.tool_callsr   r   r   %mistral_common.tokens.tokenizers.baser   r   rR   r   r   r   r   r   r3   r0   r-   <module>r      s     . . 
    \ T T Wj
O1?DUWjjkj
Z	Er"; ErP#> D=.> =C\ =*
', 
'1J 
'r0   