
    cCiQl                     j   S r SSKJr  SSKrSSKJs  Jr  SSKJr  SSKJ	r	  SSK
JrJr  SSKJr  SS	KJrJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJrJr  SSKJr  SSKJ r J!r!  SSK"J#r#  SSK$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-J.r.  \R^                  " \05      r1 " S S\#5      r2 " S S\-5      r3 " S S\5      r4 " S S\Rj                  5      r6 " S S\%5      r7 " S S\.5      r8 " S  S!\&\5      r9 " S" S#\,5      r: " S$ S%\+5      r; " S& S'\'5      r< " S( S)\)5      r= " S* S+\*5      r> " S, S-\(5      r?/ S.Qr@g)/zPyTorch MiniMax model.    )OptionalN)nn   )ACT2FN)CacheDynamicCache)layer_type_validation)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)MoeModelOutputWithPast)Unpack)TransformersKwargslogging)deprecate_kwarg)OutputRecordercheck_model_inputs   )MixtralConfig)
MixtralAttentionMixtralDecoderLayerMixtralForCausalLMMixtralForQuestionAnswering MixtralForSequenceClassificationMixtralForTokenClassificationMixtralModelMixtralPreTrainedModelMixtralRMSNormMixtralSparseMoeBlockc                   @   ^  \ rS rSrSr        SU 4S jjrSrU =r$ )MiniMaxConfig5   an  
This is the configuration class to store the configuration of a [`MiniMaxModel`]. It is used to instantiate an
MiniMax model according to the specified arguments, defining the model architecture. Instantiating a configuration
with the defaults will yield a similar configuration to that of the MiniMax.

[MiniMaxAI/MiniMax-Text-01-hf](https://huggingface.co/MiniMaxAI/MiniMax-Text-01-hf)

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.


Args:
    vocab_size (`int`, *optional*, defaults to 32000):
        Vocabulary size of the MiniMax model. Defines the number of different tokens that can be represented by the
        `inputs_ids` passed when calling [`MiniMaxModel`]
    hidden_size (`int`, *optional*, defaults to 4096):
        Dimension of the hidden representations.
    intermediate_size (`int`, *optional*, defaults to 14336):
        Dimension of the MLP representations.
    num_hidden_layers (`int`, *optional*, defaults to 32):
        Number of hidden layers in the Transformer encoder.
    num_attention_heads (`int`, *optional*, defaults to 32):
        Number of attention heads for each attention layer in the Transformer encoder.
    num_key_value_heads (`int`, *optional*, defaults to 8):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details, check out [this
        paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `8`.
    head_dim (`int`, *optional*, defaults to `hidden_size // num_attention_heads`):
        The attention head dimension.
    hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string) in the decoder.
    max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
        The maximum sequence length that this model might ever be used with. MiniMax's sliding window attention
        allows sequence of up to 4096*32 tokens.
    initializer_range (`float`, *optional*, defaults to 0.02):
        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    rms_norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the model should return the last key/values attentions (not used by all models). Only
        relevant if `config.is_decoder=True`.
    pad_token_id (`int`, *optional*):
        The id of the padding token.
    bos_token_id (`int`, *optional*, defaults to 1):
        The id of the "beginning-of-sequence" token.
    eos_token_id (`int`, *optional*, defaults to 2):
        The id of the "end-of-sequence" token.
    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
        Whether the model's input and output word embeddings should be tied.
    rope_theta (`float`, *optional*, defaults to 1000000.0):
        The base period of the RoPE embeddings.
    sliding_window (`int`, *optional*):
        Sliding window attention window size. If not specified, will default to `4096`.
    attention_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the attention probabilities.
    num_experts_per_tok (`int`, *optional*, defaults to 2):
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter
    num_local_experts (`int`, *optional*, defaults to 8):
        Number of experts per Sparse MLP layer.
    output_router_logits (`bool`, *optional*, defaults to `False`):
        Whether or not the router logits should be returned by the model. Enabling this will also
        allow the model to output the auxiliary loss. See [here]() for more details
    router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
        The aux loss factor for the total loss.
    router_jitter_noise (`float`, *optional*, defaults to 0.0):
        Amount of noise to add to the router.
    layer_types (`list`, *optional*):
        Attention pattern for each layer.
    block_size (`int`, *optional*, defaults to 256):
        The length of each attention block, determining how queries, keys, and values
        are grouped and processed for intra- and inter-block attention.
    full_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after normal attention.
    full_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after normal attention.
    linear_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after lightning attention.
    linear_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after lightning attention.
    mlp_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after MLP.
    mlp_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after MLP.

```python
>>> from transformers import MiniMaxModel, MiniMaxConfig

>>> # Initializing a MiniMax style configuration
>>> configuration = MiniMaxConfig()

>>> # Initializing a model from the MiniMax style configuration
>>> model = MiniMaxModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```c	                 p  > [         TU ]  " S0 U	D6  Xl        X l        X0l        X@l        XPl        X`l        Xpl        Xl	        U R                  cC  [        U R                  5       V
s/ s H  n
[        U
S-   S-  5      (       a  SOSPM     sn
U l        [        U R                  U R                  5        g s  sn
f )N   r   full_attentionlinear_attention )super__init__layer_types
block_sizefull_attn_alpha_factorfull_attn_beta_factorlinear_attn_alpha_factorlinear_attn_beta_factormlp_alpha_factormlp_beta_factorrangenum_hidden_layersboolr	   )selfr+   r,   r-   r.   r/   r0   r1   r2   super_kwargsi	__class__s              e/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/minimax/modular_minimax.pyr*   MiniMaxConfig.__init__   s     	(<(&$&<#%:"(@%'>$ 0.#W\]a]s]sWt WtRSD!a%1$5$5 ;MMWt D 	d..0F0FG s   &$B3)r,   r-   r.   r+   r/   r0   r1   r2   )N   r%   r%   r%   r%   r%   r%   )__name__
__module____qualname____firstlineno____doc__r*   __static_attributes____classcell__r9   s   @r:   r"   r"   5   s0    cN  !" !H H    r"   c                       \ rS rSrSrg)MiniMaxRMSNorm   r(   Nr=   r>   r?   r@   rB   r(   rE   r:   rG   rG          rE   rG   c                      ^  \ rS rSrU 4S jrS rS\4S jrU 4S jrS\4U 4S jjr	S r
S	\4S
 jrS\R                  4S jrS\4S jrSrU =r$ )MiniMaxCache   c                 0   > [         TU ]  5         / U l        g N)r)   r*   linear_cacher6   r9   s    r:   r*   MiniMaxCache.__init__   s    02rE   c                     [        [        U R                  5      US-   5       H  nU R                  R                  / 5        M      X R                  U'   g )Nr%   )r3   lenrP   append)r6   	layer_idxrP   _s       r:   set_linear_cacheMiniMaxCache.set_linear_cache   sD    s4,,-y1}=A$$R( >'3)$rE   rV   c                 @    U[        U 5      :  a  U R                  U   $ g rO   )rT   rP   r6   rV   s     r:   get_linear_cacheMiniMaxCache.get_linear_cache   s"    s4y $$Y//rE   c                 Z   > [        [        TU ]	  5       [        U R                  5      5      $ rO   )maxr)   __len__rT   rP   rQ   s    r:   r`   MiniMaxCache.__len__   s"    57?$c$*;*;&<==rE   c                    > U[        U R                  5      :  a#  U R                  U   / :w  a  U R                  U   4$ [        TU ]  U5      $ rO   )rT   rP   r)   __getitem__)r6   rV   r9   s     r:   rc   MiniMaxCache.__getitem__   sM    s4,,--$2C2CI2NRT2T%%i022w"9--rE   c              #   N   #    [        [        U 5      5       H	  nX   v   M     g 7frO   )r3   rT   r[   s     r:   __iter__MiniMaxCache.__iter__   s      s4y)I/! *s   #%repeatsc                     [        [        U 5      5       H`  nU R                  U   / :w  a,  U R                  U   R                  USS9U R                  U'   MB  U R                  U   R                  U5        Mb     g )Nr   dim)r3   rT   rP   repeat_interleavelayersbatch_repeat_interleave)r6   rh   rV   s      r:   rn   $MiniMaxCache.batch_repeat_interleave   sl    s4y)I  +r1/3/@/@/K/]/]^ekl/]/m!!),I&>>wG	 *rE   indicesc                     [        [        U 5      5       HW  nU R                  U   / :w  a#  U R                  U   US4   U R                  U'   M9  U R                  U   R	                  U5        MY     g )N.)r3   rT   rP   rm   batch_select_indices)r6   rp   rV   s      r:   rr   !MiniMaxCache.batch_select_indices   sd    s4y)I  +r1/3/@/@/KGUXL/Y!!),I&;;GD	 *rE   
max_lengthc                     [        S5      e)Nz*MiniMaxCache doesnot support `crop` method)RuntimeError)r6   rt   s     r:   cropMiniMaxCache.crop   s    GHHrE   )rP   )r=   r>   r?   r@   r*   rX   intr\   r`   rc   rf   rn   torchTensorrr   rw   rB   rC   rD   s   @r:   rL   rL      sc    34# 
>.S .
"Hs HEELL EIs I IrE   rL   c                   d  ^  \ rS rSrS\S\4U 4S jjrS rS r\	" SSS	S
9  SS\
R                  S\\
R                  \
R                  4   S\\
R                     S\\   S\\
R                     S\\   S\\
R                  \\
R                     \\\
R                        4   4S jj5       rSrU =r$ )MiniMaxLightningAttention   configrV   c                   > [         TU ]  5         X l        [        USS 5      =(       d    UR                  UR
                  -  U l        UR
                  U l        UR                  U l        UR                  U l        [        UR                     U l        [        U R                  U R
                  -  5      U l        [        R                  " UR                  U R
                  U R                  -  S-  SS9U l        [        R                  " U R
                  U R                  -  UR                  SS9U l        [        R                  " UR                  U R
                  U R                  -  SS9U l        U R'                  5       nU R)                  U5      u  pEnU R+                  SU5        U R+                  SU5        U R+                  SU5        U R+                  SU5        g )	Nhead_dimr   F)bias
slope_ratequery_decay	key_decaydiagonal_decay)r)   r*   rV   getattrhidden_sizenum_attention_headsr   r4   r,   r   
hidden_actact_fnrG   normr   Linearqkv_projout_projoutput_gateget_slope_ratedecay_factorsregister_buffer)r6   r   rV   r   r   r   r   r9   s          r:   r*   "MiniMaxLightningAttention.__init__   s   "
D9mV=O=OSYSmSm=m#)#=#= !'!9!9 ++V../"4==43K3K#KL			&"4"4d6N6NQUQ^Q^6^ab6bino		$":":T]]"JFL^L^ejk99V%7%79Q9QTXTaTa9ahmn((*
151C1CJ1O.\:6]K8[)4-~>rE   c                     SSSU R                   -  -  -  n[        R                  " U R                   5      S-   nSU R                  U R                  S-
  S-   -  -
  S-   nX-  nXC-  nUS S 2S S 4   nU$ )Nr%   r      gh㈵>)r   rz   arangerV   r4   )r6   baseexponentfactorrates        r:   r   (MiniMaxLightningAttention.get_slope_rate  s    A!d66678<< 8 89A=T^^t'='='AD'HIIDP~}AtTM"rE   c                    [         R                  " U R                  5      S-   n[         R                  " U* US S 2S 4   -  5      n[         R                  " U* U R                  US S 2S 4   -
  -  5      nUS S 2S 4   US S S 24   -
  nUS S S S 2S S 24   nX-  n[         R                  " US:  U* [        S5      5      n[         R                  " U5      nX4U4$ )Nr%   r   z-inf)rz   r   r,   expwherefloat)r6   r   block_size_ranger   r   r   s         r:   r   'MiniMaxLightningAttention.decay_factors  s     <<81<ii.>q$w.G GHIIzkT__?OPQSWPW?X-XYZ	)!T'25EdAg5NN'dAq(89#4^q%8>/5QW=Y>2~55rE   past_key_valuepast_key_values4.58new_nameversionhidden_statesposition_embeddingsattention_maskcache_positionkwargsreturnc                 	   UR                   u  pxn	XR                  -   S-
  U R                  -  n
U R                  U R                  U5      5      nUR	                  XxU R
                  SU R                  -  5      n[        R                  " XR                  SS9u  pnUR                  SS5      nUR                  SS5      nUR                  SS5      nS nUb  UR                  U R                  5      nUGc  [        R                  " XpR
                  U R                  U R                  5      R                  U5      nUbN  UR                  [        R                  S9nUR                  UR!                  S5      R!                  S5      ) S5      n/ n[#        U
5       GHh  nUU R                  -  n[%        UU R                  -   U5      nUU-
  nUS S 2S S 2UU24   nUS S 2S S 2UU24   nUS S 2S S 2UU24   nU R&                  S S 2S U24   nU R(                  S S 2U* S 24   nU R*                  S S 2S S 2S U2S U24   n[        R,                  " U R.                  * U-  5      n[        R0                  " UUR                  SS5      5      n[        R0                  " UU-  U5      n[        R0                  " UU-  U5      nUU-   nUR3                  U5        [        R0                  " UU-  R                  SS5      U5      n UU-  U -   nGMk     O[        R,                  " U R.                  * 5      n!/ n[#        U5       H  nUS S 2S S 2UUS-   24   nUS S 2S S 2UUS-   24   nUS S 2S S 2UUS-   24   n[        R0                  " UR                  SS5      U5      n"U!U-  U"-   n[        R0                  " UU5      nUR3                  U5        M     [        R4                  " USS9nUR                  SS5      nUR	                  XxU R
                  U R                  -  5      nU R7                  U5      n[8        R:                  " U R=                  U5      5      U-  nU R?                  U5      nUb  URA                  U R                  U5        UU4$ )	Nr%   r   rj   r   )dtyper   )!shaper,   r   r   reshaper   r   rz   split	transposer\   rV   zerostor5   masked_fill	unsqueezer3   minr   r   r   r   r   matmulrU   catr   Fsigmoidr   r   rX   )#r6   r   r   r   r   r   r   
batch_sizeseq_lenr   
num_blocks
qkv_statesquery_states
key_statesvalue_statesattn_weights_interattn_outputr8   	start_idxend_idxcurrent_block_sizecurrent_query_statescurrent_key_statescurrent_value_statescurrent_query_decaycurrent_key_decaycurrent_diagonal_decayblock_decayattn_weights_intraattn_output_intraattn_output_intercurrent_attn_outputnext_attn_weights_interratiocurrent_attn_weights_inters#                                      r:   forward!MiniMaxLightningAttention.forward  s    ,9+>+>(
[/!3G
[[}!=>
''
T=U=UWX[_[h[hWhi
16Z\]1^.,#--a3))!Q/
#--a3 "&!0!A!A$..!Q%!&Z9Q9QSWS`S`bfbobo!p!s!s"
 )!/!2!2!2!D+779Q9QRS9T9^9^_a9b8bdefK:&/	i$//97C%,y%8"'3Aq)G:K4K'L$%/1i6G0G%H"'3Aq)G:K4K'L$&*&6&6q:M;M:M7M&N#$(NN17I6I6J3J$K!)-)<)<QCVDVCVXkYkXk=k)l&#ii(8;M(MN &+\\2FHZHdHdegikHl%m"$)LL1CF\1\^r$s! %*LL1EH[1[]o$p! '8:K&K#""#67 +0,,'*;;FFr2NPd+' &8+%EH_%_"; '@ IIt./EK7^'3Aq!a!e)O'D$%/1a!a%i%@"'3Aq!a!e)O'D$-2\\:L:V:VWY[]:^`t-u*%*-?%?B\%\"&+ll3GI[&\#""#67 $ ii4 "++Aq1!))*t?W?WZ^ZgZg?ghii,ii 0 0 ?@;NmmK0 &,,T^^=OP...rE   )
r   r,   r   rV   r   r   r4   r   r   r   )NN)r=   r>   r?   r@   r"   ry   r*   r   r   r   rz   r{   tupler   r   
LongTensorr   r   r   rB   rC   rD   s   @r:   r}   r}      s    ?} ? ?,	6 %0A6R ,059`/||`/ #5<<#=>`/ !.	`/
 "%`/ !!1!12`/ -.`/ 
u||Xell3XeELL>Q5RR	S`/ S`/rE   r}   c                       \ rS rSrSrg)MiniMaxAttentioni~  r(   NrI   r(   rE   r:   r   r   ~  rJ   rE   r   c                       \ rS rSrSrg)MiniMaxSparseMoeBlocki  r(   NrI   r(   rE   r:   r   r     rJ   rE   r   c                     ^  \ rS rSrS\S\4U 4S jjr\" SSSS9       SS	\R                  S
\
\R                  \R                  4   S\\R                     S\\R                     S\\   S\\   S\\   S\\   S\\R                     S\\   S\
\R"                  \\
\R"                  \R"                  4      4   4S jj5       rSrU =r$ )MiniMaxDecoderLayeri  r   rV   c                   > [         TU ]  X5        X l        UR                  U   U l        UR
                  U l        UR                  U l        U R                  S:X  a3  [        X5      U l        UR                  U l
        UR                  U l        g [        X5      U l        UR                  U l
        UR                  U l        g )Nr'   )r)   r*   rV   r+   
layer_typer1   r2   r}   	self_attnr/   attn_alpha_factorr0   attn_beta_factorr   r-   r.   )r6   r   rV   r9   s      r:   r*   MiniMaxDecoderLayer.__init__  s    +" ,,Y7 & 7 7%55??006vIDN%+%D%DD"$*$B$BD!-f@DN%+%B%BD"$*$@$@D!rE   r   r   r   r   r   r   r   position_idsoutput_attentionsoutput_router_logits	use_cacher   r   r   c
                 &   U R                  U5      nUnU R                  " SUUUUUUUU	S.U
D6u  pXR                  -  XR                  -  -   nU R	                  U5      nUnU R                  U5      u  pXR                  -  XR                  -  -   nU$ )a  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.
    attention_mask (`torch.Tensor`, *optional*): attention mask of size
        `(batch, sequence_length)` where padding elements are indicated by 0.
    past_key_values (`Cache`, *optional*): cached past key and value projection states
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    output_router_logits (`bool`, *optional*):
        Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
        should not be returned during inference.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
        Indices depicting the position of the input sequence tokens in the sequence.
    kwargs (`dict`, *optional*):
        Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
        into the model
)r   r   r   r   r   r   r   r   r(   )input_layernormr   r   r   post_attention_layernormblock_sparse_moer1   r2   )r6   r   r   r   r   r   r   r   r   r   r   residualrW   s                r:   r   MiniMaxDecoderLayer.forward  s    N ,,];   >> 

' 3)%+/)

 

 !#9#99MLaLa<aa 55mD 00? #8#88=K_K_;__rE   )r   r   rV   r   r1   r2   r   )NNNFFFN)r=   r>   r?   r@   r"   ry   r*   r   rz   r{   r   r   r   r   r5   r   r   FloatTensorr   rB   rC   rD   s   @r:   r   r     s2   A} A A" %0A6R
 2637+/,1/4$)59=||= #5<<#=>= !.	=
 u//0= "%= $D>= 'tn= D>= !!1!12= -.= 
u  (51B1BEDUDU1U+V"WW	X= S=rE   r   c                   2    \ rS rSrSr\" \SS9\\\	/S.r
Srg)MiniMaxPreTrainedModeli  Fr%   )index)router_logitsr   
attentionsr(   N)r=   r>   r?   r@   _can_compile_fullgraphr   r   r   r   r}   _can_record_outputsrB   r(   rE   r:   r   r     s'    "'(=QG,')BCrE   r   c                      \ rS rSr\" 5               SS\\R                     S\\R                     S\\R                     S\\	   S\\R                     S\\   S	\\   S
\\R                     S\\   S\4S jj5       rSrg)MiniMaxModeli  N	input_idsr   r   r   inputs_embedsr   r   r   r   r   c	                    US L US L-  (       a  [        S5      eU(       a  Uc  [        5       nO4U(       a-  [        U[        5      (       d  [        S[        U5       S35      eUc  U R	                  U5      nUcD  Ub  UR                  5       OSn
[        R                  " XUR                  S   -   UR                  S9nUc  UR                  S5      nU R                  R                  c  [        O[        nU" U R                  UUUUUS9nUnU R                  X5      nU R                    H(  nUR"                  S:X  a  UnOUnU" U4UUUUUUS	.U	D6nM*     U R%                  U5      n['        UUS
9$ )Nz:You must specify exactly one of input_ids or inputs_embedszSMiniMax uses cache of its own and is not compatible with `past_key_values` of type .r   r%   )device)r   input_embedsr   r   r   r   r&   )r   r   r   r   r   r   )last_hidden_stater   )
ValueErrorrL   
isinstancetypeembed_tokensget_seq_lengthrz   r   r   r  r   r   sliding_windowr
   r   
rotary_embrm   r   r   r   )r6   r   r   r   r   r   r   r   r   r   past_seen_tokensmask_functioncausal_maskr   r   decoder_layerinput_attention_masks                    r:   r   MiniMaxModel.forward  s    -t";<YZZ0*nOz/<HHefjkzf{e||}~    --i8M!CRC^==?de"\\ ]5H5H5K"KTaThThN )33A6L.2kk.H.H.P*Vw#;;&))+%
 & #oomJ![[M''+;;'2$ (6$)	$73) /#-	 	M )$ 		-0%++
 	
rE   r(   )NNNNNNNN)r=   r>   r?   r@   r   r   rz   r   r{   rL   r   r5   r   r   r   r   rB   r(   rE   r:   r   r     s     1515372659$(,059G
E,,-G
 !.G
 u//0	G

 ",/G
   1 12G
 D>G
 $D>G
 !!1!12G
 +,G
 
 G
 G
rE   r   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )MiniMaxForCausalLMi.  c                 $   > [         TU ]  " S0 UD6$ )a   
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MiniMaxForCausalLM

>>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r(   )r)   r   )r6   r7   r9   s     r:   r   MiniMaxForCausalLM.forward/  s    . w...rE   r(   )r=   r>   r?   r@   r   rB   rC   rD   s   @r:   r  r  .  s    / /rE   r  c                       \ rS rSrSrg) MiniMaxForSequenceClassificationiI  r(   NrI   r(   rE   r:   r  r  I  rJ   rE   r  c                       \ rS rSrSrg)MiniMaxForTokenClassificationiM  r(   NrI   r(   rE   r:   r  r  M  rJ   rE   r  c                       \ rS rSrSrg)MiniMaxForQuestionAnsweringiQ  r(   NrI   r(   rE   r:   r  r  Q  rJ   rE   r  )r"   r   r   r  r  r  r  )ArA   typingr   rz   torch.nn.functionalr   
functionalr   activationsr   cache_utilsr   r   configuration_utilsr	   masking_utilsr
   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   processing_utilsr   utilsr   r   utils.deprecationr   utils.genericr   r   mixtral.configuration_mixtralr   mixtral.modeling_mixtralr   r   r   r   r   r   r   r   r   r    
get_loggerr=   loggerr"   rG   rL   Moduler}   r   r   r   r   r   r  r  r  r  __all__r(   rE   r:   <module>r0     s!          ! . 8 R B 9 6 & 0 0 ? 9   
		H	%@HM @HF	^ 	+I< +I\Q/		 Q/h	' 		1 	P-/I Pf3 I
< I
X/+ /6	'G 		$A 		"= 	rE   