
    oi_                        S SK 7  S SKJr  SSKJrJr  SSKJr  SSKJ	r	  SSK
JrJrJrJrJr  S S	KJrJrJr   SS
KJrJrJrJrJrJrJr  SSKJr   SSKJ r J!r!  \"(       a  SSK#J$r$         S#S\%RL                  S\'\(   S\'\%RL                     S\'\%RR                     S\'\*\%RL                        S\+S\+S\'\%RR                     S\*\%RL                  \'\%RL                     \'\*\%RL                        4   4S jjr,       S#S\%RL                  S\'\(   S\'\%RL                     S\'\%RR                     S\'\*\%RL                        S\'\+   S\'\+   S\'\%RR                     4S jjr-SSK.J/r0  Sr1\%Rd                  Rf                  Rh                  r5\%Rl                  r7\%Rp                  r9   S$S\%RL                  S\'\*\%RL                        4S jjr: S%S  jr; " S! S"\<5      r=g!   \" \5      r\\" S5      :  d  \" S\ S35      e GN= f!   \r \r! GN= f)&   )*)__version__    )
_get_dtypeVersion)dtype_from_config   )get_packed_info_from_kwargs)AttentionConfigAttentionContextrun_attentionselect_attention_backendSDPA)GemmaFixedRotaryEmbedding&GemmaFixedLinearScalingRotaryEmbeddingfast_geglu_inference)Gemma2AttentionGemma2DecoderLayerGemma2ModelGemma2ForCausalLMGemma2RotaryEmbeddingapply_rotary_pos_emb	repeat_kvz4.42z&Unsloth: Your transformers version of z does not support Gemma2.
The minimum required version is 4.42.3.
Try `pip install --upgrade "transformers>=4.42.3"`
to obtain the latest transformers build, then restart this session.)*_prepare_4d_causal_attention_mask_for_sdpa)Gemma2SdpaAttentionGemma2FlashAttention2)flash_attn_funcNhidden_statescausal_maskattention_maskposition_idspast_key_valueoutput_attentions	use_cachepadding_maskreturnc	                 f   [        U S5      (       a  U ?U ?U ?U ?U ?U ?U ?UR                  5       u  pnU R                  R                  nU R                  nU R                  R                  nU R                  nUU-  U:X  d   eU R                  X5      u  nnnUR                  XUU5      R!                  SS5      nUR                  XUU5      R!                  SS5      nUR                  XUU5      R!                  SS5      n[#        U
UR$                  5      nUR&                  S   nUb  UUS   R&                  S   -  nUR$                  R(                  nU R*                  R,                  U   nU R*                  R.                  U   nUb  UOU
R1                  S5      nUb2  U R*                  R3                  UU5      u  nn[5        UUUUU5      u  nnO[5        UUUU5      u  nnUb2  [6        R9                  US   U/SS9n[6        R9                  US   U/SS9nU(       a  UU4OS nU
R1                  S5      nUb  UO[;        U[<        5      =(       a    US	L n[>        =(       a    US L nU(       Ga  S
n [A        U R                  SS 5      n!U(       a  U!b  U!OUn!UU!::  a  S
OU!U!4n [        U S5      (       d!  SU R                  RB                  S-  -  U l"        US L=(       a    US L n"[G        [I        U"5      UUS	U R                  RJ                  U RD                  U S.SU RD                  S	U R                  RJ                  U S.S9n#[M        UUUUUURN                  UUUU!S9
n$[Q        U#U$UUUS9n%U%RS                  XUU-  5      n%OSU
;   a  [T        O[V        n&U&" UUUX UU5      n%U RY                  U U%5      n%U%S U4$ )Npaged_attentionr   r	   r   r!   )dimuse_sliding_windowT)r,   sliding_window_flash_attention_softmax_scale      ?g      ?)causalsoftcapsoftmax_scalewindow_sizeg        )	dropout_pr2   r0   r1   r3   )backend
n_kv_headsn_groupsflash_dense_kwargsflash_varlen_kwargs)
bszq_len
kv_seq_lenn_headshead_dimrequires_gradseq_infor    r   r-   )configcontextQKV_flag_for_generation)-hasattrpaged_attention_Kpaged_attention_Vr(   temp_QAtemp_KVRH_Q	attentionsizerA   num_attention_headsnum_key_value_groupsnum_key_value_headsr>   	apply_qkvview	transposer
   deviceshapeindex
rotary_embmulti_gpu_cos_cachedmulti_gpu_sin_cachedget
get_cachedfast_rope_embeddingtorchcat
isinstanceboolHAS_FLASH_ATTENTION_SOFTCAPPINGgetattrquery_pre_attn_scalarr.   r   r   attn_logit_softcappingr   r?   r   reshape$slow_inference_attention_softcappingslow_attention_softcappingapply_o)'selfr   r   r    r!   r"   r#   r$   r%   argskwargsr:   r;   _r=   r7   r6   r>   rC   rD   rE   r@   r<   device_indexcossinrope_position_idscos_varsin_varr+   has_sliding_window	use_flashwindowr-   
use_varlenattention_configrB   Afxs'                                          O/home/james-whalen/.local/lib/python3.13/site-packages/unsloth/models/gemma2.pyGemma2Attention_fast_forwardr|   H   s    t&''"" LLIN!&&(MCkk--G((H00J}}H G+++nnT1GAq!	s7H-771=A	s:x0::1a@A	s:x0::1a@A*6188<HJ!nQ'--b11
88>>L
//
.
.|
<C
//
.
.|
<C %0fjj6P  $??55j,O"1a';LM1"1ac21!II~a(!,AI6II~a(!,AI6(aVdN  $89 ) 	T*B{d/B  0JNd4JI .>E"0"<* 
 / $n5  t=>>2511363D/ T)Dn.D
*.z:#;;==!%!D!D%	" !!%!D!D;;==%#
& ##)77+%+
 #3wASTZ[\IIc'H"45 &/ 1+ 	
 q!Q3
;T1AdN""    c	                 T   U(       a  [        U S5      (       a  [        R                  U R                  R                  R
                  [        R                  SS9nUn[        U R                  X5      nU R                  " SUUUUUUUUU R                  S.	U
D6u  pn[        U R                  X5      nX-  nUn[        U R                  X5      n[        U R                  U5      n[        U R                  X5      nX-  nOUn[        U R                  USS9nU R                  " SUUUUUUUUS.U
D6u  pn[        U R                  USS9nX-   nUn[        U R                  USS9nU R                  U5      n[        U R                  USS9nX-   nU4nU(       a  X4-  nU(       a  X4-  nU$ )	NrF   zcuda:0dtyperU   )	r   r   r    r!   r"   r#   r$   r%   rF   T)gemma)r   r   r    r!   r"   r#   r$   r%    )rG   r^   emptyinput_layernormweightrV   float32"fast_rms_layernorm_inference_gemma	self_attnrF   post_attention_layernormpre_feedforward_layernormr   mlppost_feedforward_layernormfast_rms_layernorm)rj   r   r   r    r!   r"   r#   r$   r%   rk   rl   
out_weightresidualself_attn_weightspresent_key_valueoutputss                   r{   Gemma2DecoderLayer_fast_forwardr      s    W$  [[  ''--u}}x ! 


 !:  -
 ?Cnn ?
)%+'+ 1!'#'#<#<?
 ?
;*; ;))=
 	! !:**M
 -TXX}E:++]
 	! *  -
 ?Cnn 
?
)%+'+ 1!'
?
 
?
;*; +))=$
 !0 !***M4
 /*++]D
 !0G''''Nr}   )sqrt   c           
      d   UnUR                  5       u  pnUu  pUR                  nU R                  R                  nU R                  nU R                  R
                  nU R                  nU R                  R                  nUU-  nUR                  S   nUS-   nUR                  nU(       Ga  [        R                  [        U-   S-   SU	UU4UUS9U l        U R                  S S 2S4   U l        U R                  S S 2S4   U l        UR!                  SSSS5      U R                  S U& UR!                  SSSS5      U R                  S U& [        R                  SU	SU4UUS9U l        [        R                  SU	SUU-  4UUS9U l        [        R                  XSU4UUS9U l        [        R                  SU	U4UUS9U l        [        R                  XS[        U-   4UUS9U l        S[-        U R                  R.                  5      -  U l        US-  U l        U R                  R4                  U l        SU R                  R4                  -  U l        OUU R                  R                  S   :  a  U R                  R;                  U R                  R                  S   [        -   SU	UU45        U R                  S S 2S4   U l        U R                  S S 2S4   U l        U R*                  R;                  XSU R*                  R                  S   [        -   45        [=        U R>                  XR"                  S   S	9n[=        U R@                  XR$                  S   S	9n[=        U RB                  XR$                  S   S	9nURE                  U	SUU5      RG                  SS5      nURE                  U	SUU5      RG                  SS5      nURE                  U	SUU5      RG                  SS5      nU RH                  RK                  UUR                  RL                  5      u  nnUU   RO                  S5      nUU   RO                  S5      nU R2                  nU R&                  nUS S 2S S 2S S 2US 24   US S 2S S 2S S 2S U24'   US S 2S S 2S S 2S U24   US S 2S S 2S S 2US 24'   [        RQ                  US S 2S S 2S S 2S U24   US S 2S S 2S S 2S U24   S	9  UU-  nURS                  UU5        US S 2S U2S S 2S S 24   nUS S 2S S 2S S 2US 24   US S 2S S 2S S 2S U24'   US S 2S S 2S S 2S U24   US S 2S S 2S S 2US 24'   [        RQ                  US S 2S S 2S S 2S U24   US S 2S S 2S S 2S U24   S	9  UU-  nURS                  UU5        UR!                  SSSS5      U R                  U'   UR!                  SSSS5      U R                  U'   U R                  S U R!                  SSSS5      nU R                  S U R!                  SSSS5      nU R                  RT                  n U(       a.  UU :  a(  SU -
  n!US S 2S S 2U!S 2S S 24   n"US S 2S S 2U!S 2S S 24   n#OUUn#n"U"R                  u    n
n$n
US:w  ap  U"S S 2S S 2S S S 2S S 24   RW                  U	UUU$U5      n"U#S S 2S S 2S S S 2S S 24   RW                  U	UUU$U5      n#U"RY                  XU$U5      n"U#RY                  XU$U5      n#UU R0                  -  n[[        UU"RG                  SS5      U R*                  S S 2S S 2S S 2S U$24   S	9n%U%U R8                  -  n%[]        U%U%S	9  U%U R6                  -  n%[_        U%S[        R`                  S
9U%S S & [[        U%U#US	9n%U%RG                  SS5      n%U%RY                  U	SU5      n%[=        U Rb                  U%U R(                  S	9n%U%UU44$ )Nr)   r   r	   r   r      r/   r,   )out)r*   r   )2rN   r   rA   rO   rP   rQ   r>   hidden_sizerV   rU   r^   r   KV_CACHE_INCREMENTr(   rH   rI   permuterJ   rK   rL   temp_OrM   	math_sqrtrd   scalarhalf_head_dimre   treciprocal_tresize_fast_linear_forwardq_projk_projv_projrS   rT   rX   r\   rW   	unsqueezenegaddcmul_r-   expandrf   torch_matmul
torch_tanhtorch_nn_functional_softmaxr   o_proj)&rj   r   r"   r!   
do_prefillr    r+   rl   Xnr:   rm   hdK1V1r   r=   r7   r6   r>   r   attention_sizeseq_lenr<   rU   QnKnVnro   rp   hrL   RH_Kr-   slicing_tokensKnnVnn
cached_lenry   s&                                         r{   &Gemma2Attention_fast_forward_inferencer   4  s    
B##%JCBFBHHEkk--G((H00J}}H ++))Kx'NhhrlG1J!!F ${{')A-q#z8L  +  

 "&!5!5ad!;!%!5!5ad!;+-::aAq+Ax(+-::aAq+Ax({{Q' # 
 {{Q
X-. # 
 KKq( ;UU[K\	kk1c;"7QWkX107:;UU[ % 
 Idkk&G&GHH%]33$++"D"DD	t++11!4	4$$$$**1-0BB	
 "&!5!5ad!;!%!5!5ad!;1dnn22269KKL	
 
T[["LLO	DB	T[["LLO	DB	T[["LLO	DB	a(	+	5	5a	;B	aX	.	8	8A	>B	aX	.	8	8A	>B ))*biiooFHC
l

%
%a
(C
l

%
%a
(CA99D1aAB;DAq"1"1aBQB;DAq!"	IId1aBQB;tAq!RaRK'8I9#IBKKc	;J;1D 1aAB;DAq"1"1aBQB;DAq!"	IId1aBQB;tAq!RaRK'8I9#IBKKc
 ')jjAq!&<D7#&(jjAq!&<D7#				,	4	4Q1a	@B				,	4	4Q1a	@B [[//Nj>9^+A~)*A~)*rS ))Aq*a1}!Qa"#**Xz8
 !Qa"#**Xz8
 kk#
H=kk#
H= B 	Rq!,DNN1aKZKCW4XYA 		AqKA&qEMMJAaDQ2&A 	
AqA			#q.)ADKK$++>Ar2h;r}   c                   ^  [        U 4S j[        [        5       5       5      nUS S 2S T R                  24   nT R                  R                  U5      nUR                  [        [        T R                  5      5      5      nU[        R                  [        T R                  R                  5      UR                  S9-  nUR                  u  pn
US   S   R                  S   nUS:w  aB  [         (       a  SnSnO6[#        UX4UUT R                  R$                  S9n[#        UX4UU5      nOUnUn/ n['        T R                  R(                  5       GH  u  nn[+        US	S5      n[-        UXs5      u  psUS
-  S:H  nUn[/        UR0                  XvU   5      n[3        UR4                  UX/   UU(       a  UOU[7        UR4                  S5      (       + US9u  nn[/        UR8                  UUU   5      nUU-  nUn[/        UR:                  UUU   5      n[=        UR>                  U5      n[/        UR@                  UUU   5      nUU-  nURC                  U5        GM	     [/        T R                  RD                  XvW   5      n[G        UU/ / S9$ )Nc              3      >#    U  Hf  n[         R                  TR                  R                  S    R                  R
                  [         R                  [         R                  U5      S9v   Mh     g7f)r   r   N)r^   
empty_likemodellayersr   r   r   rU   ).0xrj   s     r{   	<genexpr>5Gemma2Model_fast_forward_inference.<locals>.<genexpr>  s\       %A 	JJa 0077MM\\!_ 	 	

 %s   A.A1)r   r   r)   r   TF)r-   _per_layer_device_indexr	   r(   )r   r"   r!   r    r   r+   )last_hidden_statepast_key_valuesr   
attentions)$tuplerangeDEVICE_COUNTmax_seq_lengthr   embed_tokenstor   r   rA   r^   tensorr   r   r   rV   rb   r   r-   	enumerater   rc   move_to_devicer   r   r   r   rG   r   r   r   r   r   appendnormBaseModelOutputWithPast)rj   	input_idsr   r!   r    rl   out_weightsr   r:   r;   r   r   SWAGAnext_decoder_cacheidxdecoder_layerrn   r+   r   r   s   `                    r{   "Gemma2Model_fast_forward_inferencer     s      |$ K !2t22223IJJ++I6M!$$Z0A$++0N%OPM U\\$++))*M4G4G "  M #((NCa #))"-G
ax**CB<!%!;!;C <	B '

(9(9:] }.GK&4-'
# !1W\ :))=l:S
 ,R##),1'$6SB$]%<%<>OPP!3,
(( ;22%

 	! :33%

 -]->->N:44%

 	!!!"34Y ;Z 7

L(AM #),	 r}   c                   4    \ rS rSr\S 5       r\S 5       rSrg)FastGemma2ModeliD  c                     [        S[        [        [        S9u  pU b(  [	        U[        5       5        [        U 5      [        l        [        [        l	        [        [        l	        [        [        l	        [        [        l	        [        [        l	        [!        ["        5      [$        l	        [&        [(        l	        [+        [$        5        SS Kn[        UR.                  R0                  R2                  l        g )Ngemma2)
model_namerope_modulescaled_rope_moduleattention_moduler   )patch_linear_scalingr   r   r   execglobalseval__init__r|   forwardr   r   r   r   LlamaModel_fast_forwardr   CausalLM_fast_forwardr   r   PeftModel_fast_forwardPeftModelForCausalLM!fix_prepare_inputs_for_generation*transformers.models.gemma2.modeling_gemma2modelsr   modeling_gemma2r   )	init_namefunctiontransformerss      r{   	pre_patchFastGemma2Model.pre_patchE  s    2!3!G.	
	  79%'+IO$">&B#(D%%D"5$9.%
! (>$)*;< 	: & 	""22H 	r}   c                    [        XSS9u  pSSKJn  U R                  5        H5  u  p4SU;   d  SU;   a  UR	                  S5        M$  UR	                  S5        M7     U R                  5        H;  u  p5[        XR5      (       d  M  [        US5      (       a  M*  UR                  Ul	        M=     SS K
n[        S	5       H1  nUR                  5         [        R                  R                  5         M3     X4$ )
NF)downcast_roper   )Gemma2RMSNormz.lora_A.z.lora_B.Tvariance_epsilonr   )patch_model_and_tokenizerr   r   named_parametersrequires_grad_named_modulesr`   rG   epsr   gcr   collectr^   cudaempty_cache)r   	tokenizerr   nameparammoduler  rm   s           r{   
post_patchFastGemma2Model.post_patchg  s     5e
 	M !113KDT!Z4%7$$T*$$U+	 4 "//1LD&00 v'9::

 + 2 	qAJJLJJ""$  r}   r   N)__name__
__module____qualname____firstlineno__staticmethodr   r  __static_attributes__r   r}   r{   r   r   D  s)     B &  & r}   r   )NNNNFFN)FNF)N)>llama_utilsr   unsloth_zoo.utilsr   r   unsloth_zoo.hf_utilsr   utils.packingr
   utils.attention_dispatchr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   transformers_versionImportError%transformers.modeling_attn_mask_utilsr   r   r   rb   
flash_attnr   r^   TensorOptionalBlockDiagonalCausalMask
LongTensorTuplera   r|   r   mathr   r   r   nn
functionalsoftmaxr   matmulr   tanhr   r   r   FastLlamaModelr   r   r}   r{   <module>r+     s     1 2 7  
  &
, #* 6:-1/348#/3D#<<D# 12D# U\\*	D#
 5++,D# U5<<01D# D# D# 5++,D# 5<<%,,/%:M1NNOD#V 6:-1/348(- %/3Y<<Y 12Y U\\*	Y
 5++,Y U5<<01Y  ~Y ~Y 5++,Yx # #hh1199 ||ZZ
 a<<a U5<<01aV gTJ n J s
"#7876?245I4J KR S
 	
 3$,)+s   G H "G?H