
    oi                     t    S SK r S SKJr  S SKJrJr  SSKJr  SSKJ	r	J
r
Jr  S r \R                  " \5        g)    N)OptionalTuple   )TEMPORARY_PATCHES)patch_functionKWARGS_TYPEraise_errorc                    ^^  SS K mSSK Jm  U4S jn    SS[        R
                  S[        [        R
                     S[        [        [        R
                  [        R
                  4      S[        [           S	[        S
[        [        R
                  [        [        R
                     4   4U4S jjjn[        TR                  R                  R                  R                  SU5        [        TR                  R                  R                  R                  SU5        g ! [         a  n [        SU 5      s S n A $ S n A ff = f)Nr   )apply_rotary_pos_embPixtralAttentionc                   > [         TR                  R                  R                  R                  U ]  5         Xl        UR                  U l        UR                  U l
        U R                  U R                  -  U l        U R                  S-  U l        UR                  U l        [        R                   " U R                  U R                  SS9U l        [        R                   " U R                  U R                  SS9U l        [        R                   " U R                  U R                  SS9U l        [        R                   " U R                  U R                  SS9U l        g )Ng      F)bias)supermodelspixtralmodeling_pixtralr   __init__confighidden_size	embed_dimnum_attention_heads	num_headshead_dimscaleattention_dropoutdropoutnnLineark_projv_projq_projo_proj)selfr   transformerss     _/home/james-whalen/.local/lib/python3.13/site-packages/unsloth_zoo/temporary_patches/pixtral.pyr   (patch_PixtralAttention.<locals>.__init__"   s    l!!))::KKT[]++33$..8]]D(
//iiUKiiUKiiUKiiUK    hidden_statesattention_maskposition_embeddingsoutput_attentionskwargsreturnc           	      *  > UR                  5       u  pgnU R                  U5      n	U R                  U5      n
U R                  U5      nU	R	                  XgU R
                  U R                  5      R                  SS5      n	U
R	                  XgU R
                  U R                  5      R                  SS5      n
UR	                  XgU R
                  U R                  5      R                  SS5      nUu  pT" XXSS9u  p[        R                  R                  R                  U	U
UUU R                  (       a  U R                  OSU R                  S9nUR                  SS5      R                  5       nUR!                  XgS5      nU R#                  U5      nUS 4$ )Nr      r   )unsqueeze_dimg        )	attn_mask	dropout_pr   )sizer!   r   r    viewr   r   	transposetorchr   
functionalscaled_dot_product_attentiontrainingr   r   
contiguousreshaper"   )r#   r(   r)   r*   r+   r,   
batch_sizepatches_query_states
key_statesvalue_statescossinattn_outputr   s                  r%   forward'patch_PixtralAttention.<locals>.forward2   sg    "/!3!3!5
Q{{=1{{=1
{{=1#((dnndmm\ffghjkl!((dnndmm\ffghjkl
#((dnndmm\ffghjkl&#7RUjk#l hh))FF&(,3

 G 
 "++Aq1<<>!))*rBkk+.D  r'   r   rF   )NNF),transformers.models.pixtral.modeling_pixtralr   	Exceptionr	   r7   Tensorr   tupleboolr   r   r   r   r   r   r   )er   rF   r   r$   s      @@r%   patch_PixtralAttentionrN      s   2;UL 	
 26KO,1"!||"! !."! &eELL%,,,F&GH	"!
 $D>"! "! 
u||Xell33	4"! "!H ##44EE ##44EE{  2-q112s   
D# #
E-D>8E>E)r7   torch.nnr   typingr   r   commonr   utilsr   r   r	   rN   append r'   r%   <module>rU      s8   "   " % EL    / 0r'   