
    oi(                         S SK r S SKJrJr  S SKrS SKJr  S SKJs  Jr	  SSK
Jr   " S S\R                  5      r " S S\5      r " S	 S
\5      rg)    N)OptionalUnion   )TRANSFORMERS_MODEL_CONFIGc                   R   ^  \ rS rSrSr\R                  4S\S\4U 4S jjjr	Sr
U =r$ )_BaseAdaptedAttention   zEBase module, which defines adaption prompts for multiple model types.
model_typeadapter_lenc           
        > [        U[        5      (       a  [        S5      e[        TU ]  5         Xl        X0l        X l        [        UR                  5       5      R                  n[        U R                  S5      (       a  U R                  R                  nO U R                  R                  R                  n[        U R                  S5      (       a  U R                  R                  U l        O%U R                  R                  R                  U l        [         R"                  " [$        R&                  " SX&XTS9R)                  5       5      U l        [         R"                  " [$        R,                  " SXTS95      U l        g)a  
Initialize object.

Args:
    model_type: The transformer model type. This is used to retrieve the right method to
        compute query states.
    adapter_len: The length of the adaption prompt to insert.
    model: The original transformer attention module that is being wrapped.
z)Unable to stack multiple adaption promptshidden_size	num_headsr   )devicedtypeN)
isinstancer   
ValueErrorsuper__init__r
   modelr   next
parametersr   hasattrr   configr   num_attention_headsnn	Parametertorchemptynormal_adaption_promptzerosadaption_gate)selfr
   r   r   target_dtyper   r   	__class__s          [/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/adaption_prompt/layer.pyr   _BaseAdaptedAttention.__init__   s    e233HII$
& e&&()00 4::}--**00K**++77K4::{++!ZZ11DN!ZZ..BBDN!||KK;FW__a 
  \\%++a*[\    )r   r"   r    r   r
   r   )__name__
__module____qualname____firstlineno____doc__r   float32strintr   __static_attributes____classcell__r%   s   @r&   r   r      s)    ONSmm (]3 (]S (] (]r(   r   c                   b  ^  \ rS rSrSrU 4S jr       SS\\\R                        S\\\R                        S\\R                     S\\R                     S\\R                     S	\\R                     S
\\   S\\   S\\\R                  \\R                     4   S4   4S jjrSrU =r$ )AdaptedAttentionGPTG   zEThis module wraps a GPT2Attention module and injects adaption promptsc                   > UR                   R                  R                  [        R                  [        R
                  4;  a   UR                   R                  R                  O[        R                  n[        TU ]!  XX4S9  g N)r$   )	c_projweightr   r   int8uint8r.   r   r   r#   r
   r   r   r$   r%   s        r&   r   AdaptedAttentionGPT.__init__J   _    ).)<)<)B)B5::W\WbWbJc)cELL%%iniviv 	 	%Sr(   hidden_states
layer_pastattention_mask	head_maskencoder_hidden_statesencoder_attention_mask	use_cacheoutput_attentionsreturn.c	                 (   U R                   " S
UUUUUUUS.U	D6n
 U
S   nU
SS  n[        U R                     R                  nUR                  S   nUR                  S   nUR                  S   n[        U R                   U5      " U R                  5      R                  USS9u  nnnUR                  SU R                  U R                  U R                   R                  5      R                  USSS5      R                  SS5      nUR                  SU R                  U R                  U R                   R                  5      R                  USSS5      R                  SS5      n[        U R                     R                  nU" U R                   XS9nUR                  n[         R"                  " UUR                  SS5      R%                  U5      5      [&        R(                  " U R                   R                  5      -  nU R*                  [,        R.                  " US[         R0                  S	9R%                  U5      -  n[         R"                  " UU5      R                  SS5      R3                  XS5      nUU-   nUR%                  U5      nU4U-   nU$ )N)r@   rB   rC   rD   rE   rF   rG   r   r      dim)r@   rD      rL   r    )r   r   r
   k_proj_layershapegetattrr    splitviewr   r   head_dimrepeat	transposecompute_query_statesr   r   matmultomathsqrtr"   Fsoftmaxr.   reshape)r#   r@   rA   rB   rC   rD   rE   rF   rG   kwargsattn_outputsattn_outputadd_outputsc_attn_layerbszq_len	embed_dim_keyvalue	adapter_k	adapter_vrY   query_statesprevious_dtypescoresadapter_outputhidden_stateoutputs                                r&   forwardAdaptedAttentionGPT.forwardP   su    zz 	
')"7#9/	
 	
	 #1o"12&0ANN"!!!$%%a(	

L9$:N:NOUUV_efUg3 HHQ(($..$**:M:MNUUVY[\^_abcmmnoqrs 	 JJq$**DNNDJJ<O<OPWWX[]^`acdeoopqstu 	  9I^^+JJm
 &++lI,?,?1,E,H,H,XY\`\e\eJJ]
 

 ##aiiBemm&T&W&WXf&ggfi8BB1aHPPQT]_` #^3 $~6 ;.r(   rP   )NNNNNFF)r)   r*   r+   r,   r-   r   r   tupler   FloatTensorTensorboolr   rt   r1   r2   r3   s   @r&   r5   r5   G   s   OT 596:158<>B$),1De&7&7 89D U5<<01D !!2!23	D
 E--.D  (5D !)):): ;D D>D $D>D 
uU\\5#667<	=D Dr(   r5   c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )AdaptedAttention   zGThis module wraps a LLamaAttention module and injects adaption prompts.c                   > UR                   R                  R                  [        R                  [        R
                  4;  a   UR                   R                  R                  O[        R                  n[        TU ]!  XX4S9  g r8   )	q_projr:   r   r   r;   r<   r.   r   r   r=   s        r&   r   AdaptedAttention.__init__   r?   r(   c                    UR                  SS5      (       a  [        S5      eU R                  " S0 UD6tp#UR                  S   nUR                  S   nUR                  S   n[        U R
                     R                  n[        U R
                     R                  n[        U R
                     R                  n	U R                  R                  R                  U R                  R                  R                  -  n
Xx:X  a8  [        U R                  U5      " U R                  5      R                  USS9u  p;nOL[        U R                  U5      " U R                  5      n[        U R                  U5      " U R                  5      n[        U R                  S5      (       a  U R                  R                   nO U R                  R"                  R$                  nUR'                  SU R(                  X-  U R                  R*                  5      R-                  USSS5      R/                  SS5      nUR'                  SU R(                  X-  U R                  R*                  5      R-                  USSS5      R/                  SS5      n[0        R2                  " XSS	9n[0        R2                  " XSS	9n[        U R
                     R4                  nU" SS
U R                  0UD6nUR6                  n[0        R8                  " UUR/                  SS5      R;                  U5      5      [<        R>                  " U R                  R*                  5      -  nU R@                  [B        RD                  " US[0        RF                  S9R;                  U5      -  n[0        R8                  " UU5      R/                  SS5      RI                  XES5      nU	b  [        U R                  U	5      " U5      nUU-   nUR;                  U5      nU/UQ7$ )a  
Forward pass for the adapter which wraps the original LlamaAttention module.

"Official" paper implementation:
https://github.com/ZrrSkywalker/LLaMA-Adapter/blob/41c3546fe1997ab8a65809dc8d8f9252b19d9faf/llama/model.py#L141

Args:
    kwargs: See the original LlamaAttention module.
output_attentionFz,output_attention is not currently supported.r   r   rJ   rK   r   )repeatsrL   r   rM   rN   rO   rP   )%getNotImplementedErrorr   rR   r   r
   rQ   v_proj_layero_proj_layerk_projin_featuresout_featuresrS   r    rT   r   r   r   r   rU   r   rV   rW   rX   r   repeat_interleaverY   r   rZ   r[   r\   r]   r"   r^   r_   r.   r`   )r#   ra   rs   ri   rf   rg   rh   rQ   r   r   factorrj   rk   r   rl   rm   rY   rn   ro   rp   rq   s                        r&   rt   AdaptedAttention.forward   s>    ::(%00%&TUUZZ)&)
ll1oQLLO	0ANN0ANN0ANNJJ))TZZ->->-K-KK 	 '#DJJ=d>R>RSYYZcijYkMAE$**l3D4H4HICDJJ5d6J6JKE4::{++

,,I

))==I HHQ((9+>ATATUVCAq!Yq!_ 	 JJq$**Y-@4::CVCVWVCAq!Yq!_ 	 ++I1M	++I1M	8I^^+G$**GG%++ lI,?,?1,E,H,H,XY\`\e\eJJ]
 

 ##aiiBemm&T&W&WXf&ggfi8BB1aHPPQT]_` #$TZZ>~NN .( >*zzr(   rP   )	r)   r*   r+   r,   r-   r   rt   r1   r2   r3   s   @r&   r{   r{      s    QTL Lr(   r{   )r\   typingr   r   r   torch.nnr   torch.nn.functional
functionalr^   r   r   Moduler   r5   r{   rP   r(   r&   <module>r      sN     "     -+]BII +]\M/ M`U, Ur(   