
    oiMX                        S SK 7  SSKrS SKJr  SSKJrJr  SSKJr  SSK	J
r
  SS	KJrJrJrJrJr  S S
K JrJr  S SK7  SSKJr  SSKJr   SSKJrJrJrJr  SSK J!r!   SSKJ"r"J#r#          S'S\$RJ                  S\&\'   S\&\$RJ                     S\&\$RP                     S\&\)\$RJ                        S\*S\*S\&\$RP                     S\&\)\$RJ                  \$RJ                  4      S\)\$RJ                  \&\$RJ                     \&\)\$RJ                        4   4S jjr+        S'S\$RJ                  S\&\'   S\&\$RJ                     S\&\$RP                     S\&\)\$RJ                        S\&\*   S\&\*   S\&\$RP                     S\&\)\$RJ                  \$RJ                  4      4S jjr,SSK-J.r/  Sr0\$Rb                  Rd                  Rf                  r4\$Rj                  r6\$Rn                  r8    S(S\$RJ                  S\&\)\$RJ                        S\&\)\$RJ                  \$RJ                  4      4S  jjr9 S)S! jr: " S" S#\5      r;S$ r< " S% S&\=5      r>g!   \" \5      r\\" S5      :  d  \" S\ S35      e GN= f!   \r"\r# GN= f)*   )*    N)__version__)
_get_dtypeVersion)dtype_from_config   )get_packed_info_from_kwargs)AttentionConfigAttentionContextrun_attentionselect_attention_backendSDPA)LlamaRotaryEmbedding!LlamaLinearScalingRotaryEmbedding)
Linear4bit)GraniteAttentionGraniteDecoderLayerGraniteModelGraniteForCausalLMz4.45.0z&Unsloth: Your transformers version of z does not support Granite.
The minimum required version is 4.45.0.
Try `pip install --upgrade "transformers>=4.45.0"`
to obtain the latest transformers build, then restart this session.)*_prepare_4d_causal_attention_mask_for_sdpa)GraniteSdpaAttentionGraniteFlashAttention2hidden_statescausal_maskattention_maskposition_idspast_key_valueoutput_attentions	use_cachepadding_maskposition_embeddingsreturnc
                    [        U S5      (       a  U ?U ?U ?U ?U ?U ?U ?UR                  5       u  pnU R                  R                  nU R                  nU R                  R                  nU R                  nU R                  (       a  U R                  R                  OSnUU-  U:X  d   eU R!                  X5      u  nnnUR#                  XUU5      R%                  SS5      nUR#                  XUU5      R%                  SS5      nUR#                  XUU5      R%                  SS5      n['        UUR(                  5      nUR*                  S   nUb  UUS   R*                  S   -  nU	c   eU	u  nnUb  UOUR-                  S5      nUb  [/        UUUUU5      u  nnO[/        UUUU5      u  nnUb2  [0        R3                  US   U/SS9n[0        R3                  US   U/SS9nU(       a  UU4OS nUS L =(       a    US L=(       a    US L nUb  [4        O
[7        U5      nUU4n[9        U SS 5      n[;        UUUS	UUUS
.SUS	S.UUUS.R=                  5        V V!s0 s H  u  n n!U!c  M  U U!_M     sn!n UUS.S9n"[?        UUUUUUR@                  UUUS9	n#[C        U"U#UUUS9n$U$RE                  XUU-  5      n%U RG                  U U%5      n%S n&U%U&U4$ s  sn!n f )Npaged_attentionr   r   r	   r   )dimscalingT)causalsoftmax_scale	dropout_pwindow_sizeg        )r+   r*   r)   )	attn_maskscaler+   )r.   p)backend
n_kv_headsn_groupsflash_dense_kwargsflash_varlen_kwargssdpa_kwargsxformers_kwargs)	bszq_len
kv_seq_lenn_headshead_dimrequires_gradseq_infor   r   )configcontextQKV)$hasattrpaged_attention_Kpaged_attention_Vr%   temp_QAtemp_KVRH_Q	attentionsizer>   num_attention_headsnum_key_value_groupsnum_key_value_headsr;   trainingattention_dropout	apply_qkvview	transposer
   deviceshapegetfast_rope_embeddingtorchcatr   r   getattrr   itemsr   r<   r   reshapeapply_o)'selfr   r   r   r   r   r   r    r!   r"   argskwargsr7   r8   _r:   r2   r1   r;   r+   r@   rA   rB   r=   r9   cossinrope_position_ids
use_varlenr0   windowr*   kvattention_configr?   Aattn_outputattn_weightss'                                          P/home/james-whalen/.local/lib/python3.13/site-packages/unsloth/models/granite.pyGraniteAttention_fast_forwardrm   D   s^    t&''"" LLIN!&&(MCkk--G((H00J}}H15--AI G+++nnT1GAq!	s7H-771=A	s:x0::1a@A	s:x0::1a@A*6188<HJ!nQ'--b11
***"HC$0fjj6P  $"1ac3DE1"1ac21!II~a(!,AI6II~a(!,AI6(aVdN 	$R84#7RNd<R 
 *0H0T  *%FD)T2M&*"!	
 *
 ,&& eg	
1
  AqD
 #
1< %33'!
G 	/7qVWXA))C((:;K,,t[1KLn44A
s   
K&Kc
                    [        U S5      (       a  U R                  OU R                  R                  nU(       a  [        U S5      (       a  Un[        U R                  U5      nU R
                  " SUUUUUUUUU	U R                  S.
UD6u  pn[        R                  XUS9nUn[        U R                  U5      n[        U R                  U5      n[        R                  XUS9nOUn[        U R                  U5      nU R
                  " SUUUUUUUUU	S.	UD6u  pn[        R                  XUS9nUn[        U R                  U5      nU R                  U5      n[        R                  XUS9nU4nU(       a  UU4-  nU(       a  UU4-  nU$ )Nresidual_multiplier_flag_for_generation)
r   r   r   r   r   r   r    r!   r"   rp   alpha)	r   r   r   r   r   r   r    r!   r"    )rC   ro   r>   fast_rms_layernorm_inferenceinput_layernorm	self_attnrp   rW   addpost_attention_layernormfast_swiglu_inferencemlpfast_rms_layernorm)r]   r   r   r   r   r   r   r    r!   r"   r^   r_   ro   residualself_attn_weightspresent_key_valueoutputss                    rl    GraniteDecoderLayer_fast_forwardr      s     4.// 	  [[,,  W$  !4  -
 ?Cnn ?
)%+'+ 1!'"5#'#<#<?
 ?
;*; 		(CV	W !4))=
 .dhhF		(CV	W *4+?+?O>Bnn ?
)%+'+ 1!'"5?
 ?
;*; 		(CV	W !*4+H+H-X/		(CV	WG%''%''N    )sqrt   c           
      2   Uc   S5       eUnUR                  5       u  pnUu  pUR                  nU R                  R                  nU R                  nU R                  R
                  nU R                  nU R                  R                  nUU-  nUR                  S   nUS-   nUR                  nU(       GaC  [        R                  [        U-   S-   SU	UU4UUS9U l        U R                  S S 2S4   U l        U R                  S S 2S4   U l        UR!                  SSSS5      U R                  S U& UR!                  SSSS5      U R                  S U& [        R                  SU	SU4UUS9U l        [        R                  SU	SUU-  4UUS9U l        [        R                  XSU4UUS9U l        [        R                  SU	U4UUS9U l        [        R                  XS[        U-   4UUS9U l        US-  U l        OUU R                  R                  S   :  a  U R                  R/                  U R                  R                  S   [        -   SU	UU45        U R                  S S 2S4   U l        U R                  S S 2S4   U l        U R*                  R/                  XSU R*                  R                  S   [        -   45        [1        U R2                  XR"                  S   S	9n[1        U R4                  XR$                  S   S	9n[1        U R6                  XR$                  S   S	9nUR9                  U	SUU5      R;                  SS5      nUR9                  U	SUU5      R;                  SS5      nUR9                  U	SUU5      R;                  SS5      nUu  nnUU   UU   nnU R,                  nU R&                  nUS S 2S S 2S S 2US 24   US S 2S S 2S S 2S U24'   US S 2S S 2S S 2S U24   US S 2S S 2S S 2US 24'   [        R=                  US S 2S S 2S S 2S U24   US S 2S S 2S S 2S U24   S	9  UU-  nUR?                  UU5        US S 2S U2S S 2S S 24   nUS S 2S S 2S S 2US 24   US S 2S S 2S S 2S U24'   US S 2S S 2S S 2S U24   US S 2S S 2S S 2US 24'   [        R=                  US S 2S S 2S S 2S U24   US S 2S S 2S S 2S U24   S	9  UU-  nUR?                  UU5        UR!                  SSSS5      U R                  U'   UR!                  SSSS5      U R                  U'   U R                  S U R!                  SSSS5      nU R                  S U R!                  SSSS5      nUR                  u    n
n n
US:w  ap  US S 2S S 2S S S 2S S 24   RA                  U	UUU U5      nUS S 2S S 2S S S 2S S 24   RA                  U	UUU U5      nURC                  XU U5      nURC                  XU U5      nUU RD                  -  n[G        UUR;                  SS5      U R*                  S S 2S S 2S S 2S U 24   S	9n![I        U!S[        RJ                  S
9U!S S & [G        U!UUS	9n!U!R;                  SS5      n!U!RC                  U	SU5      n![1        U RL                  U!U R(                  S	9n!U!UU44$ )Nz:Granite model requires position embeddings to be specifiedr&   r   r	   )dtyperS   r      )out)r'   r   )'rJ   r   r>   rK   rL   rM   r;   hidden_sizerT   rS   rW   emptyKV_CACHE_INCREMENTr%   rD   rE   permuterF   rG   rH   temp_OrI   half_head_dimresize_fast_linear_forwardq_projk_projv_projrQ   rR   negaddcmul_expandr[   r(   torch_matmultorch_nn_functional_softmaxfloat32o_proj)"r]   r   r   r   
do_prefillr   use_sliding_windowr"   Xnr7   r`   hdK1V1r   r:   r2   r1   r;   r   attention_sizeseq_lenr9   rS   QnKnVnra   rb   hrH   RH_K
cached_lenri   s"                                     rl   'GraniteAttention_fast_forward_inferencer     s    	'E	CE' 
B##%JCBFBHHEkk--G((H00J}}H ++))Kx'NhhrlG1J!!F ${{')A-q#z8L  +  

 "&!5!5ad!;!%!5!5ad!;+-::aAq+Ax(+-::aAq+Ax({{Q' # 
 {{Q
X-. # 
 KKq( ;UU[K\	kk1c;"7QWkX107:;UU[ % 
 &]	t++11!4	4$$$$**1-0BB	
 "&!5!5ad!;!%!5!5ad!;1dnn22269KKL	
 
T[["LLO	DB	T[["LLO	DB	T[["LLO	DB	a(	+	5	5a	;B	aX	.	8	8A	>B	aX	.	8	8A	>B #HC< #l"3CA99D1aAB;DAq"1"1aBQB;DAq!"	IId1aBQB;tAq!RaRK'8I9#IBKKc	;J;1D 1aAB;DAq"1"1aBQB;DAq!"	IId1aBQB;tAq!RaRK'8I9#IBKKc
 ')jjAq!&<D7#&(jjAq!&<D7#				,	4	4Q1a	@B				,	4	4Q1a	@B ((Aq*a1}1dAq !((Xz8
 1dAq !((Xz8
 ZZj(;ZZj(;
 $,,BRa+4>>!Q;J;BV3WXA 'qEMMJAaDQ"%A 	
AqA			#q.)ADKK$++>Ar2h;r   c                    US S 2S U R                   24   nU R                  R                  U5      nUR                  [	        [        U R                  5      5      5      nXPR                  R                  -  n[        U S5      (       a  U R                  OU R                  R                  nUR                  u  pxn	US   S   R                  S   n
US:w  a  [        UXx4UU
5      nOS nU R                  R                  R                  U R                   UR                  R                  5      n/ n[!        U R                  R"                  5       H  u  p[%        USS5      n['        XU5      u  pSUn[)        UR*                  U5      n[-        UR.                  UX-   UU[        UR.                  S5      (       + US9u  nn[0        R3                  UXVS9nUn[)        UR4                  U5      n[7        UR8                  U5      n[0        R3                  UXVS9nUR;                  U5        M     [)        U R                  R<                  U5      n[?        UU/ / S	9$ )
Nro   r   r&   r   _per_layer_device_indexr%   )r   r   r   r   r   r"   rq   )last_hidden_statepast_key_valuesr   
attentions) max_seq_lengthmodelembed_tokenstor   r   r>   embedding_multiplierrC   ro   rT   r   
rotary_emb
get_cachedrS   index	enumeratelayersrY   move_to_devicert   ru   r   rv   rW   rw   rx   ry   rz   appendnormBaseModelOutputWithPast)r]   	input_idsr   r   r   r   ro   r7   r8   r   r   r"   next_decoder_cacheidxdecoder_layerdevice_indexr|   r~   s                     rl   #GraniteModel_fast_forward_inferencer     sI    !2t22223IJJ++I6M!$$Z0A$++0N%OPMZZ444M 4.// 	  [[,,  #((NCa #))"-G
axCL	
 **//::]1177 '

(9(9:}.GK&4'
# !4))=
 ,S##),1'+$]%<%<>OPP"5,
(( 		(M	W 422M
 .m.?.?O		(M	W!!"34; ;< 1-PM"),	 r   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )GraniteRotaryEmbeddingi  c                     > [         TU ]  US9  g )N)r>   )super__init__)r]   r>   	__class__s     rl   r   GraniteRotaryEmbedding.__init__  s    &)r   rs   )__name__
__module____qualname____firstlineno__r   __static_attributes____classcell__)r   s   @rl   r   r     s    * *r   r   c                    ^  U 4S jnU$ )Nc                 l   > UR                  SU(       a  US   OS 5      nUb  X0l        T" U /UQ70 UD6  g )Nr>   r   )rU   r>   )r]   r^   r_   r>   original_inits       rl   new_initpatched_init.<locals>.new_init  s:    
 Hd1g4@ Kd,T,V,r   rs   )r   r   s   ` rl   patched_initr     s    - Or   c                   4    \ rS rSr\S 5       r\S 5       rSrg)FastGraniteModeli   c                  .   [        S[        [        [        S9u  pU b(  [	        U[        5       5        [        U 5      [        l        [        [        l	        [        [        l	        [        [        l	        [        [        l	        [        [        l	        [!        ["        5      [$        l	        ['        [$        R                  5      [$        l        [(        [*        l	        [-        [$        5        SS Kn[        UR0                  R2                  R4                  l        g )Ngranite)
model_namerope_modulescaled_rope_moduleattention_moduler   )patch_linear_scalingr   r   r   execglobalsevalr   rm   forwardr   r   r   r   LlamaModel_fast_forwardr   CausalLM_fast_forwardr   r   r   PeftModel_fast_forwardPeftModelForCausalLM!fix_prepare_inputs_for_generation,transformers.models.granite.modeling_granitemodelsr   modeling_granite)	init_namefunctiontransformerss      rl   	pre_patchFastGraniteModel.pre_patch  s    2"0!B/	
	  79%(,Y%#@ 'D$)F&&F#6%:/&
" '33E3N3N&O#'=$)*<=; # 	##44K 	r   c                 n   [         R                  R                  R                  U R                  R
                  R                  5      U R                  l        U R                  R                  S[        05        [         R                  R                  SSS S9nU?U R                  R                  Ul        UR                  R                  S   Ul        UR                  R                  S   Ul        X l        U R                  R
                  R                  R                  5       U R                  R                  R                  5       :w  a  [         R                  R                  SSS S9nU?U R                  R
                  R                  Ul        UR                  R                  S   Ul        UR                  R                  S   Ul        X l        UR                  R                   nU R#                  5        GHs  u  pE[%        U[&        [(        45      (       aJ  UR                  nUR*                  n[-        U5      [.        L a  UUR                  R*                  S'   OX7l        UR1                  S5      (       d  [3        US5      (       d  M  [3        US5      (       a]  UR4                  R                   U:w  aC  UR4                  R7                  U5      Ul        UR8                  R7                  U5      Ul        GM  [3        US5      (       d  GM  UR:                  R                   U:w  d  GM3  UR:                  R7                  U5      Ul        UR<                  R7                  U5      Ul        GMv     SS Kn[A        S	5       H1  n	URC                  5         [         RD                  RG                  5         M3     X4$ )
Nunsloth_versionr   )biasr   r	   r   
cos_cachedshort_cos_cachedr   )$rW   nn	Embeddingfrom_pretrainedr   r   weightr>   updater   Linearlm_headrT   in_featuresout_featuresdata_ptrr   named_modules
isinstanceBnb_Linear4bitPeft_Linear4bitquant_statetypelistendswithrC   r   r   
sin_cachedr   short_sin_cachedgcrangecollectcudaempty_cache)
r   	tokenizerr   correct_dtypenamemoduler   r  r	  r`   s
             rl   
post_patchFastGraniteModel.post_patch   s    $)88#5#5#E#EKK$$++$
  	.<= ((//!Qt/4N--%nn2215&~~33A6 KK$$++446}}##,,./ hhooa4o8G"[[55<<GN").."6"6q"9G#*>>#7#7#:G #M  ,,!//1LD&>?"CDD$00$, & MM--a0
 )6%}}\**gfl.K.K6<00%%++}<(.(9(9(<(<](KF%(.(9(9(<(<](KF%V%788++11]B.4.E.E.H.H.WF+.4.E.E.H.H.WF+3 28 	qAJJLJJ""$  r   rs   N)r   r   r   r   staticmethodr   r  r   rs   r   rl   r   r      s*     < A  A r   r   )NNNNFFNN)FNFN)N)?llamaos_utilsr   unsloth_zoo.utilsr   r   unsloth_zoo.hf_utilsr   utils.packingr
   utils.attention_dispatchr   r   r   r   r   r   r   mistralbitsandbytes.nnr   r  peft.tuners.lorar  r   r   r   r   r   transformers_versionImportError%transformers.modeling_attn_mask_utilsr   r   r   rW   TensorOptionalBlockDiagonalCausalMask
LongTensorTupleboolrm   r   mathr   	math_sqrtr   r   
functionalsoftmaxr   matmulr   tanh
torch_tanhr   r   r   r   FastLlamaModelr   rs   r   rl   <module>r0     sB    	  1 2 7   8 :
  
. 6:-1/348#/3GKu5<<u5 12u5 U\\*	u5
 5++,u5 U5<<01u5 u5 u5 5++,u5 "%ell(B"CDu5 5<<%,,/%:M1NNOu5v 6:-1/348(- %/3GKM<<M 12M U\\*	M
 5++,M U5<<01M  ~M ~M 5++,M "%ell(B"CDM` # #hh1199 ||ZZ
 GKJ<<J U5<<01J "%ell(B"CDJh GT*1 *
b ~ b k
"#7878#4445I4J KR S
 	
 5$.+-s   I I- "I*-I6