
    oi                       S SK r S SKrS SKrS SKrS SKJrJrJrJr  SSK	7  SSK	J
r
  SSK	JrJr  SSK	Jr  SSK	JrJr  S	S
KJrJr  S	SKJrJrJrJr  S SKJr  S SKJr  S SKJrJr  S SKJ r J!r!J"r"  S SK#J$r$  S	SK%J&r&J'r'J(r(J)r)J*r*J+r+  \" \5      r\\" S5      :  r, S SK-J.r.  S SK0J1r1J2r2J3r3  S SK4J5r5  S	SK67  S	SK77  SSK8J9r9  S SK0J:r:J;r;J<r<J=r=   S SK0J>r>J?r?  S SKJ@r@JArAJBrBJCrCJDrD  S SKEJFrF  S SKJGrH  S SKIJJrJJKrKJLrM  S SKIJNrNJOrO  S	SKPJQrQ  S SKRrRS SKSrSS SKTrTS SKrS SKUrUS SKVrV S SKWJXrX  S SKZJr[  \\SLr]\](       a  \\R                  R                  OSr_\(S :X  a-  \ R                  R                  rb\ R                  R                  rdO,\ R                  R                  rb\ R                  R                  rdS! rfS" rgS S#KJhri  S$rj\ R                  R                  R                  rnS%\R                  ;   rpS S&KqJrrr   SZS'\rS(\sS)\sS*S4S+ jjrt S[S, jruS- rv\ R                  rx  S\S/\ R                  S0\\\ R                        4S1 jjrz\ R                  R                  R                  r| S]S2 jr}\ R                  r\ GR                   rS^S3 jrS[S4 jr\ GR                  " S.S5\S69S7 5       r        S_S/\ R                  S8\\_   S9\\ R                     S:\\ GR                     S0\\\ R                        S;\S<\S=\\ GR                     S>\\\ R                  \ R                  4      S*\\ R                  \\ R                     \\\ R                        4   4S? jjr        S_S/\ R                  S9\\ R                     S:\\ GR                     S0\\\ R                        S;\\   S<\\   S=\\ GR                     S>\\\ R                  \ R                  4      S*\\ GR                  \\\ GR                  \ GR                  4      4   4S@ jjrSA\ GR                  \ GR                  \ GR                  SB\ GR                  \ GR                  \ GR                  SC\ GR                  \ GR                  \ GR                  0r          S`SD\\ GR                     S8\\_   S9\\ R                     S:\\ GR                     SE\\\ GR                        SF\\ GR                     S<\\   S;\\   SG\\   SH\\   S*\\\24   4SI jjr\z\}4SJ jr\" 5       rSK r\ GR(                             SaSL j5       r " SM SN\ R                  GR,                  5      r " SO SP\5      r " SQ SR\ R                  GR,                  5      r " SS ST\ R                  GR,                  5      rSU r " SV SW5      rSSXKJr  \" \SY9  g!   \/" S5      r. GN= f!   \:r>\:r? GN= f!   S SKYJXrX   GN= f)b    N)OptionalTupleListUnion   )*)*patch_unsloth_smart_gradient_checkpointing)__version__importlib_version)move_to_device)#_get_inference_mode_context_manager_prepare_model_for_qat   )get_packed_info_from_kwargsmask_packed_sequence_boundaries)AttentionConfigAttentionContextrun_attentionselect_attention_backend)scaled_dot_product_attention)r
   )Version
_get_dtype)dtype_from_configadd_dtype_kwargsfix_lora_auto_mapping)SKIP_QUANTIZATION_MODULES)is_hipget_device_typeDEVICE_TYPEDEVICE_TYPE_TORCHDEVICE_COUNTALLOW_PREQUANTIZED_MODELSz4.47.1)GradientCheckpointingLayer)loggerBaseModelOutputWithPastCausalLMOutputWithPast)*_prepare_4d_causal_attention_mask_for_sdpa)FastBaseModel)LlamaAttentionLlamaDecoderLayer
LlamaModelLlamaForCausalLM)LlamaSdpaAttentionLlamaFlashAttention2)AutoTokenizerAutoModelForCausalLM"AutoModelForSequenceClassificationBitsAndBytesConfig
AutoConfig)MODEL_FOR_CAUSAL_LM_MAPPING)set_seed)
LoraConfigTaskTypeget_peft_model)PeftModelForCausalLM"PeftModelForSequenceClassification)patch_saving_functions)	get_tokenxpuc                 p    U R                  U5      nU R                  U5      nU R                  U5      nX#U4$ N)q_projk_projv_proj)selfXQKVs        N/home/james-whalen/.local/lib/python3.13/site-packages/unsloth/models/llama.pyoriginal_apply_qkvrI      s2    AAAAAA7N    c                 (    U R                  U5      nU$ r?   )o_proj)rC   rD   Os      rH   original_apply_orN      s    AAHrJ   )sqrti   
enable_gqa)ModulesToSaveWrappermoduledevice_typeoffload_devicereturnc                    [        U S5      (       d  gU R                  R                  R                  R                  nU[
        R                  :X  a  [
        R                  nU R                  R                  R                  XSS9  U R                  R                  R                  S5        U R                  R                  USS9  U R                  R                  S5        g)ap  
Offload frozen module to CPU and configure trainable copy for mixed precision training.

This function optimizes memory usage by:
1. Moving the trainable copy to the target device with appropriate precision
2. Offloading the original frozen module to CPU/disk to free VRAM
3. Converting float16 to float32 for compatibility with certain GPUs (e.g., Tesla T4)

Args:
    module: The module to configure. Must be a ModulesToSaveWrapper with a
        `modules_to_save` attribute containing trainable and original modules.
    device_type: Target device string for training (e.g., "cuda:0", "xpu:0")
    offload_device: Device to offload frozen parameters (default: "cpu")
        Note: Currently only "cpu" is supported; disk offloading is planned.

Returns:
    None (modifies module in-place)

Note:
    - Float16 weights are automatically promoted to float32 for GPU compatibility
    - Original frozen parameters are moved to CPU to reduce active VRAM usage
    - Future versions will support disk-based offloading for even larger models

See Also:
    - https://github.com/unslothai/unsloth/pull/1200 (Tesla T4 float32 requirement)
modules_to_saveNTdevicedtypenon_blocking)rY   r[   F)hasattrrW   defaultweightrZ   torchfloat16float32torequires_grad_original_module)rR   rS   rT   	new_dtypes       rH   #_offload_frozen_module_for_trainingrf      s    @ 6,--&&..55;;IEMM! MM	
""%% &  ""11$7 ~dK
))%0rJ   c           	      `   UR                  SS 5      nUGbe  [        U5      S:X  a	  S nS US'   GOM[        US5      (       a  UR                  5       S:X  a	  S nS US'   GOUR                  u  pVUS S 2S/4   nU n[        US5      (       a  [        XwR                  5      n[        US5      (       a  S[        4S jnS	UU R                  [        R                  " XfS	-   UR                  S
9UU R                  US.n U" UR                  5      (       a  UR                  US'   UR                  " U40 UD6nOJUS S 2S/4   n[        [!        S5      ::  a-  ["        R$                  " U R&                  R(                   S35        SU;   a  US   US'   UUS.UE$ !   [        SUR                   35         N= f)Npast_key_valuesr   get_seq_lengthbase_model_prefix5_prepare_4d_causal_attention_mask_with_cache_positionrU   c                      [         R                  " [         R                  " U 5      5      nSUR                  ;   $ !   [        [        S5      :  s $ = f)NrY   z4.52.0)inspect	signatureunwrap
parameterstransformers_versionr   )fnsigs     rH   needs_device_kw<_fast_prepare_inputs_for_generation.<locals>.needs_device_kw   sH    H%//r0BC'3>>99H3gh6GGGs	   8; Ar   )rY   )sequence_lengthtarget_lengthrZ   cache_position
batch_sizeconfigrh   rY   z(Unsloth: Could not inspect signature of z4.52.4a   has no `_prepare_4d_causal_attention_mask_with_cache_position` method defined in its base modeling class. Compiled forward passes will be sub-optimal. If you're writing code, see Llama for an example implementation. If you're a user, please report this issue on GitHub.ry   position_ids)	input_idsattention_mask)getlenr\   ri   shapegetattrrk   boolrZ   r_   arangerY   r{   rl   printrr   r   r$   warning_once	__class____name__)	rC   r}   r~   kwargsrh   bscache_length
base_modelru   s	            rH   #_fast_prepare_inputs_for_generationr      s    jj!2D9O"1$"O(,F$% O%566..0A5"O(,F$%(B!!bT'*I Jz#677$Z1M1MN
S H4 H ()%1!ZZ&+ll$Q&6AQAQ' #%"kk'6
&"XX  ,5+;+;x( TT&   "0B4!8'78+<<''>>223 4+ + 6!!'(8!9~(  /B:  DD  DD  CE  Fs   8&F F-c                 >    [        U S5      (       a  [        U l        g g )Nprepare_inputs_for_generation)r\   r   r   )rR   s    rH   !fix_prepare_inputs_for_generationr   "  s    v677/R, 8rJ   Fhidden_statespast_key_valuec           
         UnUR                  5       u  pxn	Uu  pUR                  nU R                  R                  nU R                  nU R                  R
                  nU R                  nU R                  R                  nUU-  nU
R                  S   nUS-   nUR                  nU(       Ga  [        R                  " [        U-   S-   SXU4UUS9U l        U R                  SS2S4   U l        U R                  SS2S4   U l        U
R!                  SSSS5      U R                  SU& UR!                  SSSS5      U R                  SU& [        R                  " SUSU4UUS9U l        [        R                  " SUSUU-  4UUS9U l        [        R                  " X}SU4UUS9U l        UU:w  a  [        R                  " SUU4UUS9U l        O U R"                  S   SS2SS2SU24   U l        [        R                  " X}S[        U-   4UUS9U l        S[-        U R                  5      -  U l        US-  U l        OUU R                  R                  S   :  a  U R                  R3                  U R                  R                  S   [        -   SUUU45        U R                  SS2S4   U l        U R                  SS2S4   U l        U R*                  R3                  X}SU R*                  R                  S	   [        -   45        [5        U R6                  X`R"                  S   S
9n[5        U R8                  X`R$                  S   S
9n[5        U R:                  X`R$                  S   S
9nUR=                  USUU5      R?                  SS5      nUR=                  USUU5      R?                  SS5      nUR=                  USUU5      R?                  SS5      nU R@                  RC                  UUS-   5        U R@                  RE                  UUR                  RF                  5      u  nnUU   RI                  S5      nUU   RI                  S5      nU R0                  nU R&                  nUSS2SS2SS2US24   USS2SS2SS2SU24'   USS2SS2SS2SU24   USS2SS2SS2US24'   USS2SS2SS2SU24   RK                  5         UU-  nURM                  UU5        USS2SU2SS2SS24   nUSS2SS2SS2US24   USS2SS2SS2SU24'   USS2SS2SS2SU24   USS2SS2SS2US24'   USS2SS2SS2SU24   RK                  5         UU-  nURM                  UU5        UR!                  SSSS5      U R                  U'   UR!                  SSSS5      U R                  U'   U R                  SU R!                  SSSS5      nU R                  SU R!                  SSSS5      n[O        U R                  SS5      nUb.  UU:  a(  SU-
  nUSS2SS2US2SS24   n USS2SS2US2SS24   n!OUUn!n U R                  u    nn"nUS:X  d  [P        (       dt  US:w  an  U SS2SS2SSS2SS24   RS                  XUU"U5      n U!SS2SS2SSS2SS24   RS                  XUU"U5      n!U RU                  X}U"U5      n U!RU                  X}U"U5      n!UR                  S   n#U R                  S   n$Uc	  U#U$:X  a  Sn%OSn%US:X  ak  UU R.                  -  n[W        UU R?                  SS5      U R*                  SS2SS2SS2SU"24   S
9n&[Y        U&S	[        RZ                  S9U&SS& [W        U&U!US
9n&O'[P        (       a  []        UU U!UU%SS9n&O[]        UU U!UU%S9n&U&R?                  SS5      n&U&RU                  USU5      n&[5        U R^                  U&U R(                  S
9n&U&UU44$ )a  
https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L406
Fast inference using KV cache.
QK^T can be computed in 4 chunks

[Q, q] @ [K, k].T where q, k are the new tokens.
[QK^T, Qk^T]
[qK^T, qk^T]

Since the attention mask wipes Qk^T, we just get
[QK^T,    0]
[qK^T, qk^T]

Since softmax is row-wise, we get
softmax([QK^T,    0])
softmax([qK^T, qk^T])

We then multiply by   [V]
                      [v]
softmax([QK^T,    0]) [softmax(QK^T)V] *
softmax([qK^T, qk^T]) [softmax([qK^T, qk^T]) @ [V, v]]

But notice * [softmax(QK^T)V] is just the last attention.
We just need to compute the last final row.

This means we can pass in a row of Q, but we need to
remember K and V, which are called the KV cache.
r   r   rZ   rY   Nr            ?rj   outsliding_windowTF)dimrZ   )	attn_mask	is_causalrP   )r   r   )0sizerZ   r{   num_attention_headsnum_key_value_groupsnum_key_value_headshead_dimhidden_sizer   rY   r_   emptyKV_CACHE_INCREMENTpaged_attentionpaged_attention_Kpaged_attention_Vpermutetemp_QAtemp_KVRH_Qtemp_O	attention	math_sqrtscalarhalf_head_dimresize_fast_linear_forwardr@   rA   rB   view	transpose
rotary_embextend_rope_embedding
get_cachedindex	unsqueezeneg_addcmul_r   SDPA_HAS_GQAexpandreshapetorch_matmultorch_nn_functional_softmaxra   r   rL   )'rC   r   r   r|   
do_prefillr~   Xnbsz_hdK1V1rZ   n_headsn_groups
n_kv_headsr   r   attention_sizeseq_len
kv_seq_lenrY   QnKnVncossinhr   RH_Kr   slicing_tokensKnnVnn
cached_lenQ_lenK_lenr   As'                                          rH   %LlamaAttention_fast_forward_inferencer   +  s   H 
B##%JCBFBHHEkk--G((H00J}}H ++))Kx'NhhrlG1J !!F${{')A-q#8L 

 "&!5!5ad!;!%!5!5ad!;+-::aAq+Ax(+-::aAq+Ax({{Q'
 {{Q
X-.
 KKq( ;UU[\	 [(++q#{&;UU[\DK,,q/!Q*<=DK107:;UU[
 Idmm44%]	t++11!4	4$$$$**1-0BB	
 "&!5!5ad!;!%!5!5ad!;1dnn22269KKL	
 
T[["LLO	DB	T[["LLO	DB	T[["LLO	DB	a(	+	5	5a	;B	aX	.	8	8A	>B	aX	.	8	8A	>B 	OO))"gk:))*biiooFHC
l

%
%a
(C
l

%
%a
(CA99D1aAB;DAq"1"1aBQB;DAq!"Aq"1"#IBKKc	;J;1D 1aAB;DAq"1"1aBQB;DAq!"Aq"1"#IBKKc
 ')jjAq!&<D7#&(jjAq!&<D7#				,	4	4Q1a	@B				,	4	4Q1a	@B T[[*:DAN!j>&A^+A~)*A~)*rS ))Aq*a
ax||A!Qa"#**Xz8
 !Qa"#**Xz8
 kk#
H=kk#
H= HHRLEIIbME%5.		
ax
dkka#4>>!Q;J;:N+O
 +R
! Cr*<,*%!A -C.iA 	
AqA			#q.)ADKK$++>Ar2h;rJ   c           	          UR                   u  pgn[        U R                  XS9n	Ub  X-  n	[        U R                  XS9n
[	        U	SS9n	X-  n	[        U R
                  XS S 2S S 2S U24   S9nUb  X-  nU$ )Nr   Tinplace)r   r   	gate_projup_projtorch_nn_functional_silu	down_proj)rC   rD   	temp_gatetemp_upgate_multiplierdown_multiplierr   r   r   gateupdowns               rH   fast_swiglu_inferencer     s    
 JCB t~~qBD"	T\\1	<B#DD9DJD t~~taCRCi=ID"KrJ   c                    UR                   nUc>  UR                  [        R                  5      nUR	                  5       R                  SSS9nO$UR                  U5        [        [        X#S9SSUS9  X@R                  -  nX$R                  5       -  nUc  UR                  U5      nOUR                  U5        XR                  -  nU$ )Nrj   Tkeepdimr   )r   r   )rZ   rb   r_   ra   squaremeancopy_
torch_meantorch_squarevariance_epsilonrsqrt_r^   )rC   rD   XXXX2variance	old_dtypes         rH   fast_rms_layernorm_inferencer     s    I	zTT%-- 99;##B$#7
<.d(S%%%H//
B	zEE)	AHrJ   c                 H   UR                  [        R                  5      nUR                  5       R	                  SSS9nX@R
                  -  nX4R                  5       -  nUc  U R                  S-   nOU R                  US S & US-  nX2-  nUR                  UR                  5      $ )Nrj   Tr   r   )	rb   r_   ra   r   r   r   r   r^   rZ   )rC   rD   
out_weightr   r   s        rH   "fast_rms_layernorm_inference_gemmar   0  s    	
emm	Byy{d3H%%%H//
B[[3&

1c
B55>rJ   T)	fullgraphdynamicoptionsc                 >   UR                   nUR                  5       nUR                  SSS9nX-
  nU[        R                  " UR                  5       R                  SSS9U R                  -   5      -  U R                  R                  5       -  nUR                  U5      $ )Nrj   Tr   )	rZ   floatr   r_   rsqrtr   r   r^   rb   )	layernormrD   r   r   Xbars        rH   fast_layernorm_compiledr  A  s    I		A66"6%D8D
++dkkm((t(<y?Y?YY
Z	[



 
 
"	# 
 44	?rJ   causal_maskr~   r|   output_attentions	use_cachepadding_maskposition_embeddingsc
                 ,   [        U S5      (       a  U ?U ?U ?U ?U ?U ?U ?UR                  5       u  pnU R                  R                  nU R                  nU R                  R                  nU R                  nUU-  U:X  d   eU R                  X5      u  nnnUR                  XUU5      R!                  SS5      nUR                  XUU5      R!                  SS5      nUR                  XUU5      R!                  SS5      n[#        UUR$                  5      nUR&                  S   nUb  UUS   R&                  S   -  nU	(       a  UU	S   R&                  S   ::  a  U	u  nnOEU R(                  nUR+                  UUS9  UR-                  UUR$                  R.                  5      u  nnUnUc  Ub  UR1                  S5      n[3        UUUUU5      u  nnUb4  [4        R6                  " US   U/SS9n[4        R6                  " US   U/SS9nU(       a  UU4OS nUS L=(       a    US L n[9        U5      n[;        UUUS	S
0SS
S.S9n[=        UUUUUUR>                  UUUS9	n[A        UUUUUS9n U RC                  XUU-  5      n!U RE                  U U!5      n!S n"U!U"U4$ )Nr   r   r   r   r   )r   r|   r   causalT        )	dropout_pr  )backendr   r   flash_dense_kwargsflash_varlen_kwargs)	r   q_lenr   r   r   requires_gradseq_infor~   r  )r{   contextrE   rF   rG   )#r\   r   r   r   r   r   r   r   r   r{   r   r   r   r   	apply_qkvr   r   r   rY   r   r   r   r   r   r   fast_rope_embeddingr_   catr   r   r   r  r   r   apply_o)#rC   r   r  r~   r|   r   r	  r
  r  r  argsr   r   r  r   r   r   r   r   rE   rF   rG   r  r   r   r   r   rope_position_ids
use_varlenr  r{   r  r   attn_outputattn_weightss#                                      rH   LlamaAttention_fast_forwardr"  P  s    t&''"" LLIN!&&(MCkk--G((H00J}}H G+++nnT1GAq!	s7H-771=A	s:x0::1a@A	s:x0::1a@A*6188<HJ!nQ'--b11
z-@-C-I-I!-LL&S__
((j(A((QXX^^DS$ X%9"JJ~6 q!S#/@ADAq!II~a(!,A6II~a(!,A6(aVdN %@.D*@J&z2G&-,/4@F %33'!
G 	vaQANA))C((:;K,,t[1KLn44rJ   c
                    U(       a  [        U S5      (       an  Un[        U R                  U5      nU R                  " SUUUUUUUUU	S.	UD6u  pnX-  nUn[        U R                  U5      n[        U R                  U5      nX-  nOhUn[        U R                  U5      nU R                  " SUUUUUUUUU	S.	UD6u  pnX-   nUn[        U R                  U5      nU R                  U5      nX-   nU4nU(       a  X4-  nU(       a  X4-  nU$ )a  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
        `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
_flag_for_generation)	r   r  r~   r|   r   r	  r
  r  r   )r\   r   input_layernorm	self_attnpost_attention_layernormr   mlpfast_rms_layernorm)rC   r   r  r~   r|   r   r	  r
  r  r  r  r   residualself_attn_weightspresent_key_valueoutputss                   rH   LlamaDecoderLayer_fast_forwardr/    sf   6 WT#9:: 4  -
 ?Cnn ?
)%+'+ 1!'"5?
 ?
;*; 	! !4))=
 .dhhF! *4+?+?O>Bnn ?
)%+'+ 1!'"5?
 ?
;*; !0 !*4+H+H-X/ 0G''''NrJ   ra   r`   bfloat16r}   rh   inputs_embedsoutput_hidden_statesreturn_dictc                   ^^^.^/^0 Tb  TOU R                   R                  mTSL d   eU	b  U	OU R                   R                  n	Ub  UOU R                   R                  nU
b  U
OU R                   R                  n
Ub  Ub  [        S5      eUb  UR                  u  pOUb  UR                  u  pnO[        S5      eUn[        U SS5      =(       d    ST;   n[        U S5      (       a  U(       d  XR                  :  aE  Ub  UR                  OUR                  n[        R                  " SU SU S	U R                   S
35        Ub  US S 2S U R                  24   nOUb  US S 2S U R                  2S S 24   nSnUb  US   S   R                  S   nUU-   n Ub0  UR#                  SU5      R%                  [        R                  5      nOS nUb&  UR                  S   U:w  a  UR'                  US45      nUc  U R)                  U5      nUR%                  [+        [-        U R                   5      5      5      nU R                   R.                  R1                  S5      nU R                   R.                  R1                  S5      nU R                   R.                  R1                  S5      nU R                   R.                  R1                  S5      nU R                   R.                  R1                  S5      nU R(                  R2                  R4                  nU(       a  [        R6                  " [9        U R                   R:                  5      UR<                  S9nU(       a  UU-  nOeUR4                  nUR>                  (       d  URA                  5       nSnOU(       a  URC                  S5        UU-  nU(       a  URC                  S5        [        U SS5      SL a  Ub  Uc  U(       d  U RD                  (       a  UR4                  nUR>                  (       d  URA                  5       nSnOU(       a  URC                  S5        US S 2S U R                  24   nXcR!                  S5      RG                  SS5      RG                  SS5      -  nU(       a  URC                  S5        Uc  S m.O;U RD                  (       a  S nS m.O%S m.[I        UX4UU[        U R                   SS 5      S9nUnU(       d  U(       a  U R                   RJ                  U-  nUc  U RD                  (       a  SnU	(       a  SOS nT(       a  SOS nU(       a  SOS n[        U S5      (       a  U RL                  n OS n Sn!U RN                  (       a  U RD                  (       a	  U(       d  Sn!Sn"S n#S n$U(       Gat  [P        (       a  Uc  SU l)        SU l*        GOVUb4  [I        UX4UUU R                   RV                  S9n#[I        UX4UUS S9n$Sn"GO[        U S5      (       Gd  [X        (       aJ  [[        U R                  U R                   RV                  5      U l)        []        U R                  5      U l*        OU R                  n%SSK/J0n&  U&" SU R                   RV                  S9Rc                  SU%U%UR<                  [        S9Re                  S5      Re                  S5      U l)        U&" SS9Rc                  SU%U%UR<                  [        S9Re                  S5      Re                  S5      U l*         [f        (       a9  [        U S 5      (       d/  [        U Rh                  S   Rj                  S 5      (       a  U(       au  U Rl                  Ro                  UU R                   Rp                  5        U Rl                  Rs                  U R                   Rp                  URt                  Rv                  5      m0OS m0[y        U Rh                  5       GH  u  n'n(U	(       a  UU4-  nUb  UU'   OS m/Un)U(       a?  U'S-  S:H  n*U*(       a  U"(       a  U RR                  OU#n)OU"(       a  U RT                  OU$n)U*TS!'   U!(       aX  [{        U([|        5      (       dC  UUU.U/U04S" jn+[        R~                  R                  R                  U+" U(5      UU)UUSSS#9n,U,S   nOU(" U4U)UUT/TUT.T0S$.TD6n,U,S   nU(       a  UU,T(       a  SOS   4-  nT(       d  M  UU,S   4-  nGM
     U(       aC  U(       a  [        U R                  U5      nOpU(       a  [        O[        " U R                  U5      nOKU(       a  U R                  U5      nO2U(       a  [        U R                  UUS%9nO[        U R                  UUS%9nU	(       a  UU4-  nU(       a  UOS n-U
(       d  [        S& UU-UU4 5       5      $ [        UU-UUS'9$ )(NFz]Unsloth: You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timezNUnsloth: You have to specify either decoder_input_ids or decoder_inputs_embeds _unsloth_allow_packed_overlengthpacked_seq_lengthsmax_seq_lengthzUnsloth: Input IDs of shape z with length z& > the model's max sequence length of zR.
We shall truncate it ourselves. It's imperative if you correct this issue first.r   r   r   rj   r   gemmagemma2coheregranite	falcon_h1rZ   T_has_no_labelsr   r   r%  "_gradient_checkpointing_boundariesSWA_mask)AttentionMaskConverter)r   r   )r   r   use_sliding_windowc                     >^  UU UUUU4S jnU$ )Nc                  (   > T" / U QTPTP7TTS.TD6$ )N)r  r  r%  )inputsr   rR   r	  r  r   r  s    rH   custom_forwardNLlamaModel_fast_forward.<locals>.create_custom_forward.<locals>.custom_forwardN  sA    ! & * (4.A ! rJ   r%  )rR   rG  r   r	  r  r   r  s   ` rH   create_custom_forward6LlamaModel_fast_forward.<locals>.create_custom_forwardM  s      &%rJ   )use_reentrantpreserve_rng_state)r  r~   r|   r   r	  r
  r  r  )r8  c              3   0   #    U  H  nUc  M  Uv   M     g 7fr?   r%  ).0vs     rH   	<genexpr>*LlamaModel_fast_forward.<locals>.<genexpr>  s      
S ASs   	last_hidden_staterh   r   
attentions)Hr{   r	  r2  r
  use_return_dict
ValueErrorr   r   r\   r7  r$   r   r_   r   int32r    r   r   rb   repeatembed_tokensr   r   
model_type
startswithr^   r  tensorr   r   rZ   is_leafdetachrc   trainingr   r'   embedding_multiplierr@  gradient_checkpointingHAS_FLASH_ATTENTION_SOFTCAPPINGrA  GA_maskr   HAS_FLEX_ATTENTION)create_flex_attention_sliding_window_mask!create_flex_attention_causal_mask%transformers.modeling_attn_mask_utilsrB  to_causal_4dsqueezeIS_ATTENTION_REFACTORlayersr'  r   r   max_position_embeddingsr   rY   r   	enumerate
isinstancer#   utils
checkpointr   final_layernormr   normr*  tupler%   )1rC   r}   r  r~   r|   rh   r1  r
  r	  r2  r3  r  r   rz   
seq_lengthr   seq_length_with_pastallow_overlengthr   past_key_values_lengthIS_GEMMA	IS_GEMMA2	IS_COHERE
IS_GRANITEIS_FALCON_H1train_embed_tokens
normalizerinputs_requires_gradr   all_hidden_statesall_self_attnsnext_decoder_cache
boundariesra  use_static_maskdynamic_SWA_maskdynamic_GA_masknrB  idxdecoder_layermaskrC  rI  layer_outputs
next_cacher  r   r  s1           `   `                                 @@@rH   LlamaModel_fast_forwardr    s	   " ( 	[[** 
 %%%  + 	[[-- 
 '2	8M8MI #.DKK4O4O 
 !:k
 	
 
	!*
J		"$1$7$7!
\
 	
 & t%GO &  t%&&/?+++'0'<IOO-BUBUE.ug]:,Ntuy  vI  vI  uJ Jc c  !!%:t':':%:":;I&)!-Bt/B/B-BA*EFM"!0!3A!6!<!<Q!?36LL  
	!#((Z8;;EKKHa J.'..
A?L )))4!$$Z0A$++0N%OPM {{%%009H&&11(;I&&11(;I''229=J;;))44[AL**11??
 \\dkk--.8K8K

 )J6M#0#>#>  (( - 4 4 6'+$%,,U3Z'M#,,T2
 	&.$6'$#MM  -::$$)002M#' !((/'+@T-@-@+@(@A11!4>>q!DNNqRSTT((. 	
 C$"$T[[2BDI
 "M\88=H4==	 3,R$N(d t9::<<

 #""t}}Y!% OO**~/E DM DL'  J(&!%!;!;  I(&!%O $Oz**!! I'')C)C!  AATATU'' Y +$()-)C)C "\ - 3 3!2 "  WQZWQZ " +$( "\ - 3 3!2 "  WQZWQZ   	D,''4;;q>33\BB	 	--4;;>>	
 #oo88KK//1E1E1K1K
 # (4]-!111@1L-RV!$qA!(7t}}=M'6t||O+=F'(!*5+
 +
& & "KK22==%m4 $%* > M *!,M *"!/+!/$5%+&9 M *!,M=6GQ#O"QQ}Q/11Ny 5~ 8$$mM  312 ii	(M
 
		-0	*  -
 +499mXVm--'0#dJ 
#Z1BNS
 
 	

 #)$)#	 rJ   c                    ^ ^  SU U4S jjnU$ )Nc                 :  >^ US S 2S U R                   24   nUR                  u  pgU R                  R                  nU R                  R                  n	U R
                  R                  U5      n
U
R                  [        [        U R                  5      5      5      n
U
R                  u  pgnUS:X  d   e[        R                  " XgU4[        R                  [         S3S9n[        R                  " SXgU4[        R                  [         S3S9nUS   US   p[        R                  " XgS4[        R                  [         S3S9n[        R                  " SUSU	4U
R                  [         S3S9m[        U4S j[!        ["        5       5       5      [        U4S j[!        ["        5       5       5      nnUS   S   R                  S   nUS:w  a$  [%        UXg4U
U['        U R                  S	S 5      S
9nOS n/ n[)        U R
                  R*                  5       H  u  nn['        USS5      n[-        UXU5      u  pnUR/                  U
5        [1        UR2                  U
UUUS9n
T" UR4                  U
UU   UU[7        UR4                  S5      (       + S9u  n
nX-  n
UR/                  U
5        [1        UR8                  U
UUUS9n
T" UR:                  U
UU   UU   S9n
X-  n
UR=                  U5        M     [1        U R
                  R>                  U
UUUS9n
[A        U
U/ / S9$ )Nr   z:0r   r   r   c              3   t   >#    U  H-  nTS    R                  [        R                  " U5      5      v   M/     g7f)r   Nrb   r_   rY   rN  xtemp_mlps     rH   rP  g_LlamaModel_fast_forward_inference.<locals>.LlamaModel_fast_forward_inference_custom.<locals>.<genexpr>  +     O;Na(1+..a11;N   58c              3   t   >#    U  H-  nTS    R                  [        R                  " U5      5      v   M/     g7f)r   Nr  r  s     rH   rP  r    r  r  r   r   r?  _per_layer_device_index)r   r   r   r   )r   r   r|   r~   r   )r   r   rR  )!r7  r   r{   r   intermediate_sizemodelrY  rb   r   r   r_   r   ra   r    rZ   rs  ranger!   r'   r   rm  rk  r   r   r   r&  r'  r\   r(  r)  appendrr  r%   )rC   r}   rh   r|   r~   r   r   r  r   mlp_sizerD   r+  _XXr   r   r   
temp_gatestemp_upsr   r  r  r  device_indexr-  r   attention_fast_forward_inferencemlp_fast_forward_inferences                           @rH   (LlamaModel_fast_forward_inference_customT_LlamaModel_fast_forward_inference.<locals>.LlamaModel_fast_forward_inference_custom  s:    a!64#6#6!667	__
[[$$;;00JJ##I.DD-dkk:;<Bzz;;emmAR@SSU>V
 kkBDUCVVXAY
 a&#a&C;;OU]]@Q?RRT=U
 ;;Q!177@Q?RRT=U
 O5;NOOO5;NOO 

 "!$Q'--b1!8G!(6F!MN "N"+DJJ,=,=">C"=2KQOL(6a<)%A NN1,--#A $D'' !!0!5+!/!()@)@BS!TT$ A  MANN1,66#A +!!&|4"<0	A MA%%&78O #?P )JJOO
 ' !0	
 	
rJ   r?   r%  )r  r  r  s   `` rH   "_LlamaModel_fast_forward_inferencer    s     g
 g
R 43rJ   c                 &  ^               SS[         R                  S[        [           S[        [         R                     S[        [         R                     S[        [
        [         R                        S[        [         R                     S[        [         R                     S[        [           S	[        [           S
[        [           S[        [           S[        [           S[        [           S[        [        [        4   4U 4S jjjnU$ )Nr}   r  r~   r|   rh   r1  labelsr
  r	  r2  r3  num_logits_to_keeplogits_to_keeprU   c                   > Ub  T!" U UU4UUS.UD6nO[         (       a  [        R                  R                  5       OS nU	b  U	OU R                  R
                  n	U
b  U
OU R                  R                  n
Ub  UOU R                  R                  nUS L U R                  l	        U R                  " SUUUUUUUU	U
US.
UD6nUS   nUR                  u  nnnU R                  R                  nUR                  n[        U R                  SS5      n[        U R                  SS5      nUR                  n[!        X5      nUR#                  U5      nUb  UR#                  U5      n[$        R&                  R)                  SS5      S:X  a@  US:w  a  US S 2U* S 2S S 24   n[+        S UUR,                  UR.                  UR0                  S	9$ US
:X  a\  US
:X  aV  [2        R4                  " UUR7                  5       R#                  U5      5      nUR9                  S5      R9                  S5      nGOYUS:w  a/  U R                  US S 2U* S 2S S 24   R#                  U5      5      nGO$[$        R&                  R)                  SS5      S:H  nUU-  S::  a	  U(       d  SnU(       d  Ub  UR)                  SS 5      nUc  UR)                  SS 5      nU R                  R:                  S:X  a  UU R                  R<                  -  n[?        S UUS US U[        U SS 5      S SUS9nU(       d  W4US
S  -   nUb  U4U-   $ U$ [+        U[@        UR,                  UR.                  UR0                  S	9nU$  U R                  UR#                  U5      5      nUR#                  [C        [E        U R                  5      5      5      nS n[        U R                  SS5      n[        U R                  SS5      nU R                  R:                  S:X  a  S
[        U R                  SS
5      -  nO0U R                  R:                  S:X  a  U R                  R<                  nUb}  Un[2        RF                  " U5      n USS
S 24   U SS S24'   SU S'   [I        U UR)                  S5      5        UR)                  SS 5      nUc  UR)                  SS 5      n[K        UU UUUS9nOUS:w  a  URL                  (       a  UU-  nOUU-  nUS:w  aW  URL                  (       a$  SU-  U-  n[2        RN                  " U5      nUU-  nO"USU-  -  n[2        RN                  " UUS9  UU-  nU(       d  U4US
S  -   nUb  U4U-   $ U$ [+        UUUR,                  UR.                  UR0                  S	9$ )N)r|   r~   )
r}   r  r~   r|   rh   r1  r
  r	  r2  r3  r   final_logit_softcappinglogit_scaleUNSLOTH_RETURN_HIDDEN_STATES01)losslogitsrh   r   rT  r   UNSLOTH_RETURN_LOGITS   Fnum_items_in_batchn_itemsr<  accelerator_scalerT)trainerr   lm_head_weightlm_head_biasr  r  r  scaling	target_gbtorch_compilelogit_softcappingr;  logits_scaling.rj   i).rj   r6  )r  r  r  logit_scalingr  r   r   r%  )(HAS_XFORMERSxformers	attn_biasLowerTriangularMaskr{   r	  r2  rU  r  r>  r   lm_headr^   rY   r   rZ   maxrb   osenvironr   r&   rh   r   rT  r_   mvravelr   rZ  lm_head_multiplierunsloth_fused_ce_lossEMPTY_LOGITSr   r   
empty_liker   fast_cross_entropy_lossr  tanh)"rC   r}   r  r~   r|   rh   r1  r  r
  r	  r2  r3  r  r  r  r   r.  r   r   r  r   r  lm_head_devicer  r  rZ   r  RETURN_LOGITSr  r  outputshift_logitsshift_labelsfast_forward_inferences"                                    rH   _CausalLM_fast_forward5CausalLM_fast_forward.<locals>._CausalLM_fast_forward  s   $ &,  ,!/ G =IL""668d  %0 "[[22  (3 %[[55 !  +6DKK<W<W  )/$DJJ%jj %)!/+"1 -%$5';) G  
&,,UB,,%% #DKK1JAN]A> !3D &((8YY~.F ::>>8#>#E!Q& -a2D1D1Eq.H I)&")"9"9 ' 5 5$//  !8
XXg}':':'<'?'?'FGF%%a(2215F1$\\-4F3F3G0J"K"N"Nu"UVFJJNN+BCHCOMU{d"= % V%7 **%94@?$jjD9G;;))[8$1DKK4R4R$RM -"$1%,#'#%%d,@$G $$((9 #$Y4F/3/?D7V+KVK/)&-&=&=$+$9$9!(!3!3 \\-"2"25"9:F:&7&DEF#DKK1JAN]A>;;!!Y.
 5Eq IIM[[##{2 KK::M!L
 !++F3L%+CG_Lcrc"$(L!+

/0
 jj!5t<G **Y5*%%$5 -!D !''*V3Fm+F A%''!$55?F"ZZ/F.7Fc$555FJJvV4//FY,F'+'7D7V#CVC%%55#11 ++
 	
rJ   )NNNNNNNNNNNr   r   )r_   
LongTensorr   BlockDiagonalCausalMaskTensorr   FloatTensorr   intr   r   r&   )r  r  s   ` rH   CausalLM_fast_forwardr    sB    '+9=1537=A59-1$(,0/3&*,-()O
##O
 56O
 !.	O

 u//0O
 "$u'8'8"9:O
   1 12O
 ))*O
 D>O
 $D>O
 'tnO
 d^O
 %SMO
 !O
" 
u,,	-#O
 O
b "!rJ   c                     S[        [        U R                  R                  5      5      ;   nU(       a  U R                  " SUUUUUUUS.UD6$ U R                  " SUUUUUUUUU
US.
UD6$ )NClassification)r}   r~   r1  r  r	  r2  r3  )
r}   r  r~   r1  r  r	  r2  r3  r  r  r%  )strtyper   r  )rC   r}   r  r~   r1  r  r	  r2  r3  task_idsr  r  r   is_classifications                 rH   PeftModel_fast_forwardr    s      )CT__5J5J0K,LL 	
!+) 1#7%	
 	
 		
  
!%+) 1#7%!3+
 
 	
rJ   c                   V   ^  \ rS rSr     SU 4S jjrS rS	S jrS	S jrS rSr	U =r
$ )
LlamaRotaryEmbeddingi  c                 Z  > [         TU ]  5         Ubp   UR                  n[	        US5      (       a  UR
                  OSn[        USS 5      nUc"  [        UR                  UR                  -  5      n[        nUR                  nXl        X l
        X0l        [        SU R                  5      U l        S /[        -  U l        S /[        -  U l        [%        [        5       HE  nU R'                  U R                  [(        R*                  " U5      [(        R,                  " 5       S9  MG     [(        R.                  " S[1        5       [(        R,                  " 5       S	9U l        [(        R.                  " S[1        5       [(        R,                  " 5       S	9U l        g !   [        US0 5      nUS   n GN= f)
Nrope_parameters
rope_thetapartial_rotary_factorr   r      r   rY   rZ   r   rY   rZ   )super__init__r  r   r\   r  r  r   r   r    rl  r   basemincurrent_rope_sizer!   multi_gpu_cos_cachedmulti_gpu_sin_cachedr  _set_cos_sin_cacher_   rY   get_default_dtyper   get_current_device
cos_cached
sin_cached)	rC   r   rl  r  rY   r{   r  
device_idxr   s	           rH   r  LlamaRotaryEmbedding.__init__!  s    	*(( 6#:;; ,, "
 &*d3C{6--1K1KKM&F&,&D&D#'>$	!$Xt/K/K!L%)F\$9!%)F\$9!  -J##00j1//1 $  .  ++*,e6M6M6O
  ++*,e6M6M6O
E*v'8"=L)s   F F*c           
      r   Xl         SU R                  [        R                  " SU R                  S[        R
                  SS9R                  5       U R                  -  -  -  n[        R                  " U R                   S[        R
                  S9R                  5       n[        R                  " XT5      n[        R                  " Xf4SS9nUR                  5       R                  X2S	S
9nUR                  5       R                  X2S	S
9n	XR                  UR                  '   XR                  UR                  '   X4$ Nr   r   r   cpur   r  rj   r  TrZ   rY   r[   )r  r  r_   r   r   int64r  outerr  r   rb   r   r  r   r  
rC   r   rY   rZ   inv_freqtfreqsembr   r   s
             rH   r  'LlamaRotaryEmbedding._set_cos_sin_cacheT  s     ")IIQ!U[[5QWWY((
 LL""UEKK

%' 	
 A(iib1ggill5$lOggill5$lO25!!&,,/25!!&,,/xrJ   c                     Ub3  X0R                   :  a$  U R                  X1R                  UR                  S9  UR                  R                  nU R
                  U   S U U R                  U   S U 4$ Nr  r  r  rY   rZ   r   r  r  rC   r  r|   r   r  s        rH   forwardLlamaRotaryEmbedding.forwardl  sr    7-C-C#C##gRSRYRY#Zxx~~%%l3HW=%%l3HW=
 	
rJ   c                 X    Uc
  [        5       nU R                  U   U R                  U   4$ r?   r  r  r  rC   r   r  s      rH   r   LlamaRotaryEmbedding.get_cachedw  8    -/L((68Q8Q9
 
 	
rJ   c                     X R                   ::  a  g US-  US-  S:g  -   S-  U l         [        [        5       H<  nU R                  U R                   [        R
                  " U5      UR                  S9  M>     g N    r   r  r  r  r!   r  r_   rY   rZ   rC   r  r   r  s       rH   r   *LlamaRotaryEmbedding.extend_rope_embedding~  p    ,,,#*d?$17L"MQU!U-J##&&j1ISTSZSZ $  .rJ   r  r  r  r   rl  r  r  r  N   '  NNNNr   
__module____qualname____firstlineno__r  r  r  r   r   __static_attributes____classcell__r   s   @rH   r  r    s3     "&1
f0	

 rJ   r  c                   B   ^  \ rS rSrSr      SU 4S jjrS rSrU =r$ )!LlamaLinearScalingRotaryEmbeddingi  z[LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendevc                 4   > XPl         [        TU ]	  UUUUUS9  g )N)r   rl  r  rY   r{   )scaling_factorr  r  )rC   r   rl  r  rY   r+  r{   r   s          rH   r  *LlamaLinearScalingRotaryEmbedding.__init__  s,     -&= 	 	
rJ   c           
         Xl         SU R                  [        R                  " SU R                  S[        R
                  SS9R                  5       U R                  -  -  -  n[        R                  " U R                   S[        R
                  S9R                  5       nXPR                  -  n[        R                  " XT5      n[        R                  " Xf4SS9nUR                  5       R                  X2S	S
9nUR                  5       R                  X2S	S
9n	XR                  UR                  '   XR                  UR                  '   X4$ r  )r  r  r_   r   r   r  r  r+  r  r  r   rb   r   r  r   r  r  s
             rH   r  4LlamaLinearScalingRotaryEmbedding._set_cos_sin_cache  s   !(IIQ!U[[5QWWY((
 LL""UEKK

%' 	
 ###A(iib1ggill5$lOggill5$lO25!!&,,/25!!&,,/xrJ   )r  r+  )Nr  r  Nr   N)	r   r"  r#  r$  __doc__r  r  r%  r&  r'  s   @rH   r)  r)    s)    e "&
$ rJ   r)  c                   x   ^  \ rS rSr     S
U 4S jjrS rSS jrSS jrS rS\	R                  4S jrS	rU =r$ )LlamaExtendedRotaryEmbeddingi  c           
        > [         T	U ]  5         Ub_  UR                  n[        US5      (       a  UR                  OSn[        UR                  UR                  -  5      n[        nUR                  nXl
        X l	        X0l        [        SU R                  5      U l        S /[        -  U l        S /[        -  U l        SU R                  ["        R$                  " SU R                  S["        R&                  SS9R)                  5       U R                  -  -  -  nU R+                  U5      nU R-                  SUS	S
9  [/        [        5       HE  nU R1                  U R                  ["        R2                  " U5      ["        R4                  " 5       S9  MG     ["        R6                  " S[9        5       ["        R4                  " 5       S9U l        ["        R6                  " S[9        5       ["        R4                  " 5       S9U l        g )Nr  r   r  r   r   r   r   r  F
persistentr  r   r  )r  r  r  r\   r  r  r   r   r    rl  r   r  r  r  r!   r  r  r_   r   r  r  apply_scalingregister_bufferr  r  rY   r  r   r  r  r  )
rC   r   rl  r  rY   r{   r  r  r  r   s
            rH   r  %LlamaExtendedRotaryEmbedding.__init__  s    	$$D 6#:;; ,, "
 v))V-G-GGIC&F&,&D&D#'>$	!$Xt/K/K!L%)F\$9!%)F\$9! IIQ!U[[5QWWY((
 %%h/ZF  -J##00j1//1 $  .  ++*,e6M6M6O
  ++*,e6M6M6O
rJ   c                    Xl         [        R                  " U R                   U R                  R                  [        R
                  S9R                  5       n[        R                  " X@R                  5      n[        R                  " XU4SS9nUR                  5       R                  X2SS9nUR                  5       R                  X2SS9nXpR                  UR                  '   XR                  UR                  '   Xx4$ Nr  rj   r  Tr  )r  r_   r   r  rY   r  r  r  r  r   rb   r   r  r   r  )	rC   r   rY   rZ   r  r  r  r   r   s	            rH   r  /LlamaExtendedRotaryEmbedding._set_cos_sin_cache  s     ")LL""T]]-A-A5;;

%' 	
 A}}-iib1ggill5$lOggill5$lO25!!&,,/25!!&,,/xrJ   c                     Ub3  X0R                   :  a$  U R                  X1R                  UR                  S9  UR                  R                  nU R
                  U   S U U R                  U   S U 4$ r  r  r  s        rH   r  $LlamaExtendedRotaryEmbedding.forward  sr    7-C-C#C##gRSRYRY#Zxx~~%%l3HW=%%l3HW=
 	
rJ   c                 X    Uc
  [        5       nU R                  U   U R                  U   4$ r?   r  r  s      rH   r   'LlamaExtendedRotaryEmbedding.get_cached  r  rJ   c                     X R                   ::  a  g US-  US-  S:g  -   S-  U l         [        [        5       H<  nU R                  U R                   [        R
                  " U5      UR                  S9  M>     g r  r  r  s       rH   r   2LlamaExtendedRotaryEmbedding.extend_rope_embedding  r  rJ   r  c                 ~   SnSnSnSnXS-  nXT-  n/ nU H}  n	S[         R                  -  U	-  n
X:  a  UR                  U	5        M1  X:  a  UR                  X-  5        MK  Xg:w  d   eXZ-  U-
  XC-
  -  nUR                  SU-
  U	-  U-  X-  -   5        M     [        R                  " XR
                  UR                  S9$ )N   r      r  r   r   )mathpir  r_   r\  rZ   rY   )rC   r  scale_factorlow_freq_factorhigh_freq_factorold_context_lenlow_freq_wavelenhigh_freq_wavelen	new_freqsfreqwavelensmooths               rH   r5  *LlamaExtendedRotaryEmbedding.apply_scaling"  s    *<+>	D$''kD(G*  &+  !45'<<<)3oE$6   !f*!4|!Cfm!ST  ||I{{U\\RRrJ   r  r  r   )r   r"  r#  r$  r  r  r  r   r   r_   r  r5  r%  r&  r'  s   @rH   r1  r1    sF     "&6
p$

S5<< S SrJ   r1  c                   \   ^  \ rS rSr        SU 4S jjrS rS	S jrS	S jrS rSr	U =r
$ )
LongRopeRotaryEmbeddingi;  c	                   > [         TU ]  5         Uc   eUc   e[        U5      [        L d   eUb_  UR                  n[        US5      (       a  UR                  OSn	[        UR                  UR                  -  5      n[        nUR                  nXl        X l
        X0l        X@l        [        X0R                  5      U l        S /[         -  U l        S /[         -  U l        S /[         -  U l        S /[         -  U l        [*        R,                  " SU R                  S[*        R.                  SS9R1                  5       U R                  -  n
[*        R2                  " US[*        R4                  S9n[*        R2                  " US[*        R4                  S9nSXPR                  U
-  -  -  nSX`R                  U
-  -  -  nU R                  U R                  -  nUS::  a  SnON[6        R8                  " S[6        R:                  " U5      [6        R:                  " U R                  5      -  -   5      nXl        U R?                  S	US
S9  U R?                  SUS
S9  [A        5       (       a  [*        RB                  O[*        RD                  n[*        R,                  " UU RF                  RH                  [*        R.                  S9R1                  5       n[*        RJ                  " UU RF                  5      n[*        RL                  " UU4SS9n[O        [         5       H  n[*        RH                  " U5      nURQ                  5       U R<                  -  RS                  UUSS9nURU                  5       U R<                  -  RS                  UUSS9nUU R"                  U'   UU R$                  U'   M     [*        RV                  " S[Y        5       [*        RZ                  " 5       S9U l.        [*        RV                  " S[Y        5       [*        RZ                  " 5       S9U l/        [*        RV                  " S[Y        5       [*        RZ                  " 5       S9U l0        [*        RV                  " S[Y        5       [*        RZ                  " 5       S9U l1        g )Nr  r   r   r   r   r   r  r   short_inv_freqFr3  long_inv_freqrj   r  Tr  )2r  r  r  r  r  r\   r  r   r   r    rl  r    original_max_position_embeddingsr  r  r  r!   multi_gpu_short_cos_cachedmulti_gpu_short_sin_cachedmulti_gpu_long_cos_cachedmulti_gpu_long_sin_cachedr_   r   r  r  r\  ra   rD  rO   logr+  r6  is_bfloat16_supportedr0  r`   rT  rY   r  r  r  r   rb   r   r   r  r  short_cos_cachedshort_sin_cachedlong_cos_cachedlong_sin_cached)rC   r   rl  rV  r  short_factorlong_factorrY   r{   r  inv_freq_shaperT  rU  scaler+  rZ   r  r  r  r  
device_objr  r  r   s                          rH   r   LongRopeRotaryEmbedding.__init__=  s    	'''&&&45<<<$$D 6#:;; ,, "
 v))V-G-GGIC&F&,&D&D#'>$0P-	!$,.J.J"
 ,0&<*?'+/&<*?'*.,)>&*.,)>&
 LLDHHauMSSUhh 	 ||L5%--Xll;Vyy./H HI{YY-FFG ,,t/T/TTC< N!YYDHHUOdhht/T/T&UUUN - 	-~ER_m%P #8"9"9u}}LL,((//KK
 %'	 	

 At223iib1-Jj1J'')d&9&99==
4 > J '')d&9&99==
4 > J ;ED++J7:DD++J7 . !&*,e6M6M6O!
 !&*,e6M6M6O!
  %{{*,e6M6M6O 
  %{{*,e6M6M6O 
rJ   c                 ,   Xl         [        R                  " U R                   U R                  R                  [        R
                  S9R                  5       n[        R                  " X@R                  5      n[        R                  " XU4SS9nUR                  5       U R                  -  R                  X2SS9nUR                  5       U R                  -  R                  X2SS9nXpR                  UR                  '   XR                  UR                  '   Xx4$ r9  )r  r_   r   rU  rY   r  r  r  r  r   r+  rb   r   rY  r   rZ  )	rC   r   rY   rZ   r  r  r  r  r  s	            rH   r  *LongRopeRotaryEmbedding._set_cos_sin_cache  s     ")LL""T-?-?-F-FPUP[P[

%' 	
 A112iib1ggi$"5"55994 : 

 ggi$"5"55994 : 

 8B&&v||47A&&v||4%%rJ   c                 N   Ub3  X0R                   :  a$  U R                  X1R                  UR                  S9  UR                  R                  nUb3  X0R
                  :  a$  U R                  U   S U U R                  U   S U 4$ U R                  U   S U U R                  U   S U 4$ r  )
r  r  rY   rZ   r   rV  rW  rX  rY  rZ  r  s        rH   r  LongRopeRotaryEmbedding.forward  s    7-C-C#C##gRSRYRY#Zxx~~7-R-R#R//=hwG//=hwG  ..|<XgF..|<XgF rJ   c                     Uc
  [        5       nUb-  XR                  :  a  U R                  U   U R                  U   4$ U R                  U   U R
                  U   4$ r?   )r  rV  rW  rX  rY  rZ  r  s      rH   r   "LongRopeRotaryEmbedding.get_cached  su    -/L7-R-R#R22..|<= = --
)),78 	8rJ   c                     X R                   ::  a  g US-  US-  S:g  -   S-  U l         [        [        5       H<  nU R                  U R                   [        R
                  " U5      UR                  S9  M>     g r  r  r  s       rH   r   -LongRopeRotaryEmbedding.extend_rope_embedding  r  rJ   )r  r  r   r_  r`  rl  rY  rZ  rW  rX  rV  r+  r]  r^  )Ni   i   r  NNNNr   r!  r'  s   @rH   rR  rR  ;  s=     "(+/d
L&*$	8 rJ   rR  c           	         U R                   n[        R                  U 5        [        [	        U R
                  5      5      n[        U S5      (       a  [        U R
                  S5      (       a~  SU;   ax  US   br  SU;   al  US   R                  S   US   -   U R
                  R                  :  a<  [        SUS   R                  S    SUS    SU R
                  R                   S	35      eS
US'   UR                  SS 5      nUR                  SS 5      nUc  Uc  SUS'   UR                  SS 5        [        U R
                  SS 5      nUb  [        US5      (       a  US   nUR                  SU5      US'   [        U 5         [        R                  " [         US9   U R"                  " U0 UD6nS S S 5        S S S 5        U(       a  [        R%                  U 5        W$ ! , (       d  f       N4= f! , (       d  f       N== f)Nr{   rl  r}   max_new_tokensrj   zUnsloth: input length z + max_new_tokens z( exceeds the maximum sequence length of zw!
You will need to do long context extension by increasing the `max_seq_length` in `FastLanguageModel.from_pretrained`.r   cache_implementationr  r  r   token_type_idseos_token_id__iter__r   pad_token_id)rS   rZ   )r_  FastLlamaModelfor_inferencer   r   r{   r\   r   rl  rV  r   popr   r   r_   autocastr    _old_generatefor_training)	rC   r  r   restore_training_moderZ   r  r  model_eos_token_idr  s	            rH   unsloth_fast_generater~    s    !MM  &(56EtX74;;8Q#R#R6!{#/ F* {#))"-7G0HH++556 !,VK-@-F-Fr-J,KK]^deu^v]w  x`  ae  al  al  aD  aD  `E EL L  &/F!"$8$?ZZ 0$7N!n&<'(#$ JJ& !ndC%'2Dj*Q*Q/2#ZZ8JKF> 	,D1%6F##T4V4 	G 	2 ##D)M 	GF 	21s$    G,G-G,
G)	%G,,
G:c                       \ rS rSr\S 5       r\S 5       r\                     SS	 j5       r\S
 5       r\S/ SQSSSSSSSSSSS0 SSS4S j5       r	\ SS j5       r
\S 5       r\SS j5       rSrg)rv  i%  c                     [        X5      n U $ r?   )r   )r  
qat_schemes     rH   _prepare_for_qatFastLlamaModel._prepare_for_qat&  s    &u9rJ   c            	      P   [        S[        [        [        [        [
        S9u  pU b(  [        U[        5       5        [        U 5      [        l	        [        [        l        [        [        l        [        [        l        [        [        l        [         ["        l        [%        [&        5      [(        l        [*        [,        l        [/        [(        5        SS Kn[        UR2                  R4                  R6                  l        [        UR2                  R4                  R6                  l        g )Nllama)
model_namerope_modulescaled_rope_moduleextended_rope_moduleattention_modulelongrope_moduler   )patch_llama_rope_scalingr  r)  r1  r)   rR  execglobalsevalr  r"  r  r-   r.   r/  r*   r  r+   r  !LlamaModel_fast_forward_inferencer,   r  r9   r   (transformers.models.llama.modeling_llamamodelsr  modeling_llama)	init_namefunctiontransformerss      rH   	pre_patchFastLlamaModel.pre_patch+  s    6 .!B#?-5
	  79%&*9oN#!<%@"'B$$B!4
#8-$
  (>$)*:; 	8 ! 	!!00E . 	!!00R 	rJ   NTFO     c                 `   S[         R                  S'   U
(       a  U(       a  [        S5      e[        S5        U(       a  [	        5       (       d  [        S5        Sn[
        S:X  a5  [        R                  R                  5       u  nnUS:  a  [        S	5        SnO[
        S
:X  a  SnU(       a/  [         R                  R                  SS5      S:X  a  [        S5      e[        U5      nUc  [        n[        5       n[
        S:X  a  [        R                  R                  S5      nUR                  S:w  a  UR                  S-   OSn[        R                   R                  nSUR"                   SUR$                   SU S3n S['        S5       S3nGO[
        S
:X  aq  [        R                  R                  S5      nUR                  S:w  a  UR                  S-   OSn[        R                   R(                  nSU S3n S['        S5       S3nO[
        S:X  aq  [        R*                  R                  S5      nUR                  S:w  a  UR                  S-   OSn[        R                   R*                  nSU S3n S['        S5       S3nO[-        S[
         35      e[/        UR0                  S-  S-  S-  S5      nSR3                  / SP[4         PS PUR6                  S!S"  PS#P[8         PSPU PS$P[;        S%5       P[;        S%5       PS&PU PS'P[<         PS(PU PS)P[>         PS*P[;        S%5       PS+P[;        S%5       PS,P[        R4                   PSPU PS-P[@         PS.P[;        S%5       PS/P[C        U5      RE                  5        PS0P[F         PS1P[H         PS2P5      n[        U5        S3[         R                  ;   a$  [         R                  S3   n U S4;   a  Sn U S5;   a  S6n OSn U S6:X  a  [        S75        U S:w  a  S6[         R                  S3'   URK                  5         [M        UR                  S8S5      5        Uc(  U(       d  [        RN                  O[        RP                  nOAU[        RP                  :X  a-  U(       d&  [R        RT                  " S95        [        RN                  nU[        RN                  :X  d*  U[        RP                  :X  d  U[        RV                  :X  d   e[X        RZ                  " U US:S;9n!U U!l.        U!R^                  n"[a        U!5        [b        U!Rd                     n#U!Rf                  Ri                  S<5      n$Sn% [k        [l        Rn                  " U#5      S=S>S?9 n&S@U&Rq                  5       ;   n%S S S 5        Sn%Uc  U"nUcj  UU":  ad  UU"-  nU(       a  [        SA5      e[R        RT                  " SBU  SCU" SD[/        US5       SEU SF3	5        U%(       d  [        SGU  SH35      eSIUSJ.nUUSK'   S n'U(       a:  [r        Rt                  " 5       n(U$(       a  U(Rw                  SL5        [y        SSSMUU(SN9n'[{        UU"5      n)UR}                  SOS 5        U(       a  U'USP'   [        UU5      n[        5       n*Ub   [        RZ                  " U 4UUUU)U
SQSR.UD6n+GOU(       d?  [        RZ                  " U 4UUU)U
SQSS.UD6n+[        U+R                  5      U+lE        S U+lF        OSSTKGJHn,JIn-JJn.JKn/  [l        R                  " U,5      R                  n0[        U U!UUUUSUUUUSU9n1U0 H  n2U2U1;  d  M  U2U;   d  M  UU2   U1U2'   M      U," Sz0 U1D6n3U-" U3U!SV9u  n4n5U." U5U!UU'5      n+U3U+lO        U+R                  R                  U+lE        [        R                  " U/U+R                  5      U+lF        U*R                  5         U [         R                  S3'   U	c  U OU	n	[        U	U)SWUU
USX9n6[        U+U65      u  n+n6UR                  U+U65      u  n+n6[        U+R                  R                  5       H0  u  n7n8[        U8R                  l[        [        U8R                  l]        M2     SSYK^J_n9   U9R                  R6                  SZ:w  a(  [l        R                  " U9R                  5      n:U:U9lb        OU9R                  n: SS K^n;[        U;R                  5      n</ n=U< H  n>U>U:;   d  M  U=Rw                  U>5        M     [        S\S]R3                  S^ U= 5       5      -   S_-   [        5       5        [        R                  " S`U:5      R                  S5      S   n?U:R                  SaU?5      n@U:U?U@ nA[        R                  " SbUA5      R                  S5      ScS  nB[        R                  " SdU:5      R                  S5      nCSenDUDR                  S.5      nDS.R3                  UDS   /UDScS   VEs/ s H  nEWBUESfS  -   PM     snE-   5      nDU:R                  WAUD5      n:SgnDUDR                  S.5      nDS.R3                  UDS   /UDScS   VEs/ s H  nEWBUESfS  -   PM     snE-   5      nDU:R                  ShUDSc5      n:[        R                  " SiU:5      R                  S5      nC[        R                  " SjUC-   SU:[        R                  Sk9n:U:R                  SlSm5      n:U:R                  SnSZSc5      n:U:R                  SoSp5      n:[        U:[        5       5        [        U9l`        UU+lr        U+nF[        UFSq5      (       a&  UWFlr        UFR                  nF[        UFSq5      (       a  M&  UWFlr        U+R                  5        H
  nGUUGlr        M     U(       a  [        U+U6U U)SWUSr9n6[        U65         U+R                  R                  Ss[4        05        [        U+5        [        U9l`        [        U95        StU6l}        U+nI[        UISq5      (       a-  U6WIl~        SUIl        UIR                  nI[        UISq5      (       a  M-  U6WIl~        SUIl        G[         (       d  [        U+R                  Su5      (       aE  U+R                  GR                  nJU+R                  R                   H  n8WJU8R                  l        M     [        R                  " [        GR                  U+5      U+l        [        R                  " [        GR                  U+5      U+l        U+nF[        UFSq5      (       aw  [        R                  " G[        GR                  WF5      UFl        [        R                  " G[        GR                  UF5      UFl        UFR                  nF[        UFSq5      (       a  Mw  Sv[C        G[        U+5      5      ;   nKUK(       dp  U+R                  R6                  Sw:w  aV  U+R                  U+l        U+GR                  GR                  G[        l        G[        GR                  " G[        U+5      U+lD        [        GR                  " 5          U+GR                  5        H  u  nHnGG[        UG5      [        GR                  GR                  L d  M1  G[        WGSxS 5      c  MB  G[        WGSyS 5      c  MS  WGGR                   UGGR"                  GR$                  S   :  d  M  SWGGR"                  UGGR                   '   M     S S S 5        U+U64$ !   Sn GN9= f!   Sn GNC= f!   Sn GNM= f! , (       d  f       G	Nq= f!    G	Ny= f!   [        S[5      e= fs  snEf s  snEf ! , (       d  f       U+U64$ = f){Nr  UNSLOTH_USE_NEW_MODELzAUnsloth: Fast inference does not support `trust_remote_code` yet.zcUnsloth: WARNING `trust_remote_code` is True.
Are you certain you want to do remote code execution?z;Unsloth: vLLM is not installed! Will use Unsloth inference!Fcuda   zMUnsloth: vLLM does not work on older GPUs - will switch to Unsloth inference!hipTUNSLOTH_VLLM_STANDBYzjUnsloth: `unsloth_vllm_standby` is True, but  environment variable `UNSLOTH_VLLM_STANDBY` is not set to 1!r    z. zNVIDIA GPU Device. zCUDA: .z. CUDA Toolkit: z vLLM: vllmzAMD GPU Device. zROCm Toolkit: r=   zIntel XPU Device. zIntel Toolkit: z"Unsloth: Unsupported device type: r  r   z==((====))==  Unsloth z: Fast rC  z patching. Transformers: z
   \   z	   /|    zNum GPUs = z. Max memory: z GB. Platform: z.
O^O/ z_/ z    Torch: z	 Triton: 
z        /    Bfloat16 = z. FA [Xformers = z. FA2 = zA]
 "-____-"     Free license: http://github.com/unslothai/unslothHF_HUB_ENABLE_HF_TRANSFER)Falsefalse)Truetruer  zUUnsloth: Fast downloading is enabled - ignore downloading bars which are red colored!local_files_onlyz9Device does not support bfloat16. Will change to float16.sdpa)tokenattn_implementationr<  rzutf-8)encodingzself.config.rope_scalingz<Unsloth: Fast inference does not yet work with RoPE Scaling.	Unsloth: z- can only handle sequence lengths of at most z(.
But with kaiokendev's RoPE scaling of z%, it can be magically be extended to !z	However, ze doesn't support RoPE Scaling!
Please file a feature request at https://github.com/unslothai/unsloth.linear)r  factorrope_scalingout_projnf4)load_in_4bitbnb_4bit_use_double_quantbnb_4bit_quant_typebnb_4bit_compute_dtypellm_int8_skip_modulesr  quantization_configeager)
device_map
num_labelsr  rl  trust_remote_coder  )r  r  rl  r  r  )	load_vllmget_vllm_state_dictconvert_vllm_to_huggingfacegenerate_batches)r  r{   gpu_memory_utilizationr7  rZ   float8_kv_cacheenable_loramax_lora_rankdisable_log_statsuse_bitsandbytesunsloth_vllm_standby)r{   right)tokenizer_namemodel_max_lengthpadding_sider  r  fix_tokenizerTrainer_fast_inner_training_loopz3Unsloth: Unsuccessfully patched inner_training_loopz"from transformers.trainer import (z, c              3   $   #    U  H  ov   M     g 7fr?   r%  )rN  r  s     rH   rP  1FastLlamaModel.from_pretrained.<locals>.<genexpr>	  s     .:a:s   )z'logger\.info\([\"\'].+?Running trainingz

z\n([\s\t]{1,})r   z([\s\t]{1,})ao  debug_info = \
        f"==((====))==  Unsloth - 2x faster free finetuning | Num GPUs used = {len(set(p.device for p in model.parameters()))}\n"\
        f"   {chr(92)}{chr(92)}   /|    Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\n"\
        f"O^O/ {chr(92)}_/ {chr(92)}    Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\n"\
        f"{chr(92)}        /    Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size:,}\n"\
        f' "-____-"     Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_param_count(model)*100:.2f}% trained)'
        logger.warning(debug_info)
        import gc
        for _ in range(3):
            gc.collect()
            if DEVICE_TYPE == "xpu":
                torch.xpu.empty_cache()
            else:
                torch.cuda.empty_cache()rB  a3  n_total_devices = total_train_batch_size // \
            args.gradient_accumulation_steps // self._train_batch_size
        if n_total_devices > 1:
            logger.warning_once('Unsloth is running with multi GPUs - the effective batch size is multiplied by ' + str(n_total_devices))
        debug_info =zdebug_info =z
[\t\s]{1,}^)flagsz8train_dataloader = tpu_spmd_dataloader(train_dataloader)z:raise RuntimeError('Unsloth: TPUs are not yet supported!')_inner_training_loopzis_torch_tpu_available()r  r  )r  	tokenizerr  r  r  r  unsloth_versionleftr   r  r~  r^   padding_idxr%  )r  r  NotImplementedErrorr   is_vLLM_availabler   r_   r  get_device_capabilityr   RuntimeErrorhf_loginrv  r\  get_device_propertiesnameversionmajorminorr   r  r=   rV  roundtotal_memoryjoinr
   r   rr   chrr!   platform_systemtriton_versionr  upperxformers_versionHAS_FLASH_ATTENTIONr  get_statisticsr`   r0  r$   r   ra   r3   from_pretrainedr  rl   verify_fp8_support_if_applicabler4   r   rZ  r[  openrn   getfilereadr   copyr  r2   r  rx  r   RaiseUninitializedr1   r0   make_fast_generate_wrappergeneratefast_generatefast_generate_batchesunsloth_zoo.vllm_utilsr  r  r  r  getfullargspecr  dictvllm_engine	functoolspartialremoveload_correct_tokenizerpatch_tokenizer
post_patchrm  r  rk  rI   r'  r  rN   r  transformers.trainerr  r  	getsource_original_training_loopdirr  r  r  researchspanfindgroupmatchsplitreplacesub	MULTILINEr  r7  r\   modulescheck_tokenizerr;   r{   _name_or_pathendswithr   updatepatch_gradient_accumulation_fixr  _saved_temp_tokenizeris_loaded_in_8bitrj  r   r{  rw  r(   r  rz  r/  r~  types
MethodTypeno_gradnamed_modulesnn	Embeddingr   r  r^   r   )Lr  r7  rZ   r  r  r  r  r  model_patcherr  r  revisionfast_inferencer  r  random_stater  r  r  r  r  r   major_versionminor_versionSUPPORTS_BFLOAT16	gpu_statsgpu_stats_namegpu_versiongpu_stats_snippetvllm_version
max_memory
statisticsold_hf_transfermodel_configmodel_max_seq_lengthmodel_functionr|  has_rope_scalingfile
bnb_configr  rl  raise_handlerr  r  r  r  r  allowed_argsload_vllm_kwargsallowed_argllmr   quant_state_dictr  r  layerr  inner_training_loopr  items_in_trainer
good_itemsitemstartendoriginal_debugspacesfront_spaces
debug_infor  mrR   r  internal_modelr   r  sL                                                                               rH   r  FastLlamaModel.from_pretrainedR  s   2 /2

*+)W  H $&&ST!&f$/4zz/O/O/Q,} 1$g &+N%!%$JJNN#93?3F" A   *M13& 

88;I)22)=	%CX   --,,K"((99??:KK[\g[hhi j"!():6)B(C1E E!

88;I)22)=	%CU   --++K"0Q ?"!():6)B(C1E E!		77:I)22)=	%CW   --++K"1+a @"!():6)B(C1E A+OPP911D84?$FJ
O O$ O[M O O9O9OPQRT9U8V OVo O  qE  pF O  FG O  HT  GU O U Ob'Or7)O#,O-;,<O<GOHT~OUcOdncoOo~O @O  POPOG9OO #ByO(3O494E4E3FOFHOIZH[O[dOesdtOtvO 2wiO 0O 144E0F0L0L0N/OO PaO br`rO s{O |O  {POPNO 	 	j '"**4 jj)DEO"44"%"22"%!Oc!g c!69BJJ23!vzz"4e<==):EMMEenn$->K MME U]]"eu~~&=%--AW	
W "11"(

 #-+CC(6 5\5K5KL#..99+F 	goon5swOSW#=#L  P   !1N ~8L'L),@@L)R  J<'T'((Qq)**O!"!% $"
| ,] ]  !&L &2F>"
$=$B$B$D!%,,Z8+#,0&+).(=J #&n6J"K

($/ ,6F()!%0*,!6FF''*A$5&- E  (88
' *A$5&-
 
E #=U^^"LE*.E'  #11)<AAL#'%)?!/"1" -$5#/';   ,&66;&;P4:;4G$[1  ,  /./C #6cL"QA/ ,zE !$E"'"3"3"<"<E*3*;*; %"3"3+E' 	2A

./ (6'=>*+6" 1)
	 +5)<y(33E9Ey $EKK$6$67JC(:EOO%&6EOO# 8
 	1	V++448SS&-&7&78T8T&U#2E/&-&E&E# 	$|334
$D**!!$' % 	0ii.:../ I		
 		68K

$q'! "&&vu5,U37,n=CCAFqrJxx1DEKKAN,
  %%d+
YY]Oz!"~F~!v!"~~FF

 299.*U

  %%d+
YY]Oz!"~F~!v!"~~FF

 299.*VWXxx/BCII!L ff<%8",,
 299FH
 299"'

 299&
 	 '),'@$  .a!!-AA a!! *mmoF$2F! & '%'#:&I 	y)  	.<= 	u%'@$ 	(0 "(	ng..3<N0/3N,+11N ng.. 09,+/( ! GEKK$F$F//J++-7* , '..~/J/JER'//0L0LeTa!!&..}/I/I1MAN'//0K0KQOAOA a!! -DK0@@ U^^%<%<@W%W"'..E,1,?,?,G,G!)"--.CUKEN]]_ % 3 3 5f<588#5#55$7C#FM4@L!--0C0CA0FF@AFMM&*<*<= !6  iI"!"!"!H PO	D	VTUUV G GH _ is   ?{
 ;{ 6{ {: *{(>{: "A| $| |
.|
?||"|3(||
{{{%(
{72{: 7{: :{?|
|-c                      [        XSS9u  pX4$ )NT)downcast_rope)patch_model_and_tokenizer)r  r  s     rH   r  FastLlamaModel.post_patch>
  s    4d
 rJ   r@   rA   rB   rL   r   r   r   r  noneunslothr   _unsloth_temporary_saved_buffersc                    [         R                  R                  SS5      S:X  ab  S H  u  nnUU;  d  M  UUU'   M     [        R                  " SQ0 SU _SU_SU_SU_S	U_S
U_SU_SU_SU_SU	_SU
_SU_SU_SU_SU_SU_SU_UD6$ [         R                  R                  SS5      S:X  a  [        S5        U $ [        U	5        US:X  a+  [        U R                  5       R                  R                  S9  [        U5      [        La  [        S[        U5       S35      eUS::  a  [        S[        U5       S35      e[        U [         5      (       d  [        U ["        5      (       Ga  [%        U S5      (       d   eU R&                  S   R)                  5       n/ S QnS!nU H  nU=(       a    UU   [+        U5      :H  nM     [-        US   5      nUS   nUc  0 n[-        U5      nUU-  n[-        U5      [-        Ub  UO/ 5      -   n[/        U5      nU=(       a    [1        [/        U5      U-  5      S:H  nU=(       a-    U0 :H  =(       d    US L =(       a    US   0 :H  =(       d    US   S L nU(       av  [2        R4                  " S"5        S#U;   a)  [        S$5        [7        U R                  5       [8        5        S%U;   a)  [        S&5        [7        U R;                  5       [8        5        U $ [        S'5      eUc  0 n[        [<        R>                  " [@        5      5      nSU;   nSU;   nUS:w  a  [2        RB                  " S(U S)35        US*:w  a  [2        RB                  " S+U S)35        [        U5      [D        L d  US,:X  d  US-:X  d  US.:X  d  [G        S/5      eUS-:X  as  U(       d  SS K$n[K        S0URL                   S135      eU0 :X  a#  SS2K$J'n  [2        RB                  " S35        U" S4S5S69n[%        U RP                  S75      (       a  [G        S85      e[        U5      [D        L d   eU(       a$  U(       d  SS K$n[K        S0URL                   S935      e[S        S:5      n U RP                  RU                  S;[L        05        [        U5      [V        L a  [-        U5      nS<n!S<n"/ n#U Hc  n$U$S%:X  a  S!n!Uc  S%/nM  URY                  S%5        M&  U$S#:X  a  S!n"Uc  S#/nM6  URY                  S#5        MI   U$U ;   d   eU#RY                  U$5        Me     [%        U S>5      (       aM  U!(       a  U"(       d?  [        S?5        S!n!S!n"Uc  S#/nOURY                  S#5        Uc  S%/nOURY                  S%5        Ub*  U H$  n$U$S%:X  a  S!n!M  U$S#:X  a  S!n"M  [        S@U$ SA35      e   [        U[V        [,        45      (       a  [-        [/        U5      5      nS n&[%        U SB5      (       aC  U R\                  n&U R^                  n'U R`                  n(Ub  [c        SC5      eUS*:w  a  [c        SD5      eSE[        [        U 5      5      ;   n)[e        SQUUU#UUU)(       d  [f        Rh                  O[f        Rj                  UUUUUUSF.UD6n*U(       d  U*S	 U(       d  U*S	 U Rl                  n+[A        SQ0 U*D6n,U R                  5       R                  Rn                  n-U)(       a!  U Rp                  R                  Rn                  n.O$U R;                  5       R                  Rn                  n.US:X  a  U"(       a  [        SG5        [s        X5        [u        SH5       H"  n/[v        Rx                  " 5         [{        5         M$     U!(       a  [        SI5        [}        X5        [u        SH5       H"  n/[v        Rx                  " 5         [{        5         M$     [        U U,5      n [        U 5        Ub!  [        SJ5        [        R                  U U5      n U+U l6        [        R                  X5      n U(       a   U R                  5       n0U R;                  5       n1U0b  U1b  SK n2[%        U0S5      (       ax  [%        U1S5      (       ag  [%        U0R                  S5      (       aL  [%        U1R                  S5      (       a1  U2" U1R                  R                  U0R                  R                  5        [%        U0SL5      (       a.  [%        U1SL5      (       a  U2" U1R                  U0R                  5        U"(       a  [        S$5        [%        U R                  5       S5      (       d   eU R                  5       R                  R                  R                  R                  n3U3[        R                  :X  a  [        R                  n3U R                  5       R                  R                  R                  [8        U3S!SN9  U R                  5       R                  R                  R                  S!5        U!(       a  [        S&5        [%        U R;                  5       S5      (       d   eU R;                  5       R                  R                  R                  R                  n3U3[        R                  :X  a  [        R                  n3U R;                  5       R                  R                  R                  [8        U3S!SN9  U R;                  5       R                  R                  R                  S!5        U n4[%        U4S5      (       aH  [%        U4SO5      (       a  SPU4Rl                  lM        S!U4lN        U4R                  n4[%        U4S5      (       a  MH  [%        U4SO5      (       a  SPU4Rl                  lM        S!U4lN        [u        SH5       H"  n/[v        Rx                  " 5         [{        5         M$     [        U 5        [        R                  " [        R                  U 5      U lS        [        R                  " [        R                  U 5      U lT        U n5[%        U5S5      (       as  [        R                  " [        R                  U55      U5lS        [        R                  " [        R                  U55      U5lT        U5R                  n5[%        U5S5      (       a  Ms  U $ ! [Z         a'  n%U#RY                  U$5        [        S=5         S n%A%GNS n%A%ff = f! [         a$  n%[2        RB                  " SMU% 35         S n%A%GNS n%A%ff = f)RNr  r  r  ))finetune_vision_layersF)finetune_language_layersT)finetune_attention_modulesT)finetune_mlp_modulesTr  r  target_modules
lora_alphalora_dropoutbiaslayers_to_transformlayers_patternuse_gradient_checkpointingr(  r7  
use_rslorarW   init_lora_weightsloftq_configtemporary_locationensure_weight_tyingUNSLOTH_ENABLE_FULL_FINETUNINGzEUnsloth: Full finetuning is enabled, so .get_peft_model has no effectrT  r=  zUnsloth: Rank of z must be an integer.r   z must be larger than 0.peft_configr]   )r  r\  r]  r^  r_  r`  rb  rc  Tz=Unsloth: Already have LoRA adapters! We shall skip this step.rY  z>Unsloth: Training embed_tokens in mixed precision to save VRAMr  z9Unsloth: Training lm_head in mixed precision to save VRAMzQUnsloth: Your model already has LoRA adapters. Your new parameters are different.zMUnsloth: Dropout = 0 is supported for fast patching. You are using dropout = zW.
Unsloth will patch all other layers, except LoRA matrices, causing a performance hit.rS  zLUnsloth: bias = `none` is supported for fast patching. You are using bias = gaussianloftqcordazXUnsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].zUnsloth: Your PEFT version of z does not support LoftQ init.
Please install PEFT 0.7.2 or higher.
You can also install from source: `pip install git+https://github.com/huggingface/peft.git)LoftQConfigzUnsloth: init_lora_weights = `loftq` is set, but `loftq_config` is None.
We shall use `loftq_config = LoftQConfig(loftq_bits = 4, loftq_iter = 1)`.rC  r   )
loftq_bits
loftq_iterr  zUnsloth: You are using `loftq` init, yet `load_in_4bit = True` was set.
Reload your model without any quantization by setting `load_in_4bit = False`.z does not support `use_rslora`.
Please install PEFT 0.7.2 or higher.
You can also install from source: `pip install git+https://github.com/huggingface/peft.gitrR  r  Fz~Unsloth: You added custom modules, but Unsloth hasn't optimized for this.
Beware - your finetuning might be noticeably slower!_need_to_train_embeddingszUnsloth: You added new tokens but did not specify if you wanted to train the lm_head and embed_tokens.
We must turn it on for you.zUnsloth: Module = z> is not allowed. Only 'lm_head' and 'embed_tokens' is allowed.r  zTUnsloth: Currently fast inference does not work with training embeddings or lm_head.zKUnsloth: Currently fast inference does not work with using biases for LoRA.r  )r  r\  r[  r]  r^  	task_typer_  rc  rd  rb  rW   rf  z9Unsloth: Offloading input_embeddings to disk to save VRAMr   z:Unsloth: Offloading output_embeddings to disk to save VRAMz:Unsloth: Applying QAT to mitigate quantization degradationc                    [        US5      (       d  g UR                  nS[        U S0 5      ;   a  U R                  R	                  S5        [        U S5      (       a   [        U S5        U R                  SU5        g ! [         a9  n[        R                  " S[        U 5      R                   SU 35         S nANQS nAff = f)Nr^   _parametersz?Unsloth: Could not delete existing weight attr during retie on z: )r\   r^   r   rr  rx  delattr	Exceptionr$   r   r  r   register_parameter)target_modulesource_moduler^   excs       rH   _retie_parameter7FastLlamaModel.get_peft_model.<locals>._retie_parameter  s    &}h??"!.!5!5#w}mR'PP)5599(C"=(;;" 'x @ &886J $- " & 3 3&e'+M':'C'C&DBse%M!" !""s   A< <
B?/B::B?rd   zGUnsloth: Failed to ensure weight tying between embeddings and lm_head: rX   r  r  r%  )Ur  r  r   r(   r8   r   transformers_set_seedr	   get_input_embeddingsr^   rZ   r  r  	TypeErrorr  rn  r9   r:   r\   rh  to_dictr  listsetr   r$   warningrf   r    get_output_embeddingsrn   ro   r6   r   r   rV  peftr  r
   rl  r{   	frozensetr  rs  r  AssertionErrorr  r  r  r  r  r7   	CAUSAL_LMSEQ_CLSr  rY   scoreoffload_input_embeddingsr  gccollectclean_gpu_cacheoffload_output_embeddings_get_peft_modelr   rv  r  patch_peft_modelrW   r]   rd   rt  r_   r`   ra   rb   rc   r  r  r  patch_peft_fast_inferencer  r  r{  rw  )6r  r  r[  r\  r]  r^  r_  r`  ra  r(  r7  rb  rW   rc  rd  re  r  rf  r   peft_argflagrh  check_parameters	check_allparamold_target_modulesnew_target_modulesro   SUPPORTS_LOFTQSUPPORTS_RSLORAr  rl  accepted_modulestrain_lm_headr}  final_modulesrR   er  vllm_fast_generatevllm_fast_generate_batchesr  	argumentsr  lora_configinput_embeddings_deviceoutput_embeddings_devicer   input_embeddingsoutput_embeddingsry  re   rL  rK  s6                                                         rH   r8   FastLlamaModel.get_peft_modelE
  s   < ::>>1373>#$ 6)'+F8$# !//  "0 (	
  ,  ': "0 .H  , "0 ( #2 %6  ,  &8!" ':% ( ::>>:C@CGW Ll+%26224;;AA 7#/Ax7KLMM6/Ax7NOPPe122j57
 7
 5-0000++I6>>@K	  I)%M;u+=e+L	 * "&k2B&C!D)*;<O&"$"?3O/1 "&n!5#2#>B9 "
 "%%7!8! C*+.@@AQF  " #;|t'; /25 ;">2d:	  S "%77X 82246G  22UV73357H g  L))*56	'94&)31_`l_m nh i
 6>^_c^d eh i "#t+ J. G+ G+j  '!"4T5E5E4F Gq q  r!,##a  +Ju||%:;; d 
 J4'''""4T5E5E4F Gq q  %

 	.<= E)"?3O"$F"
 !%"*'0kO#**95>)
 &*""*'5&6O#**>:!%5555!((0 C %H 5566 (:W !%%)""*'5&6O#**>:"*'0kO#**95 &)Y&$(M~-)-&#,VH4rs  * ot}55"3#78O5-((++K!&!4!4).)D)D&*)j  v~)a 
 -DK0@@ 
#*'2C**IYIY"5 1'#-"5
 
	 .),' % ; ; -9-"'"<"<">"E"E"L"L',{{'9'9'@'@$','B'B'D'K'K'R'R$%2!QR(C 1X

!  RS)%D 1X

!   {3e$ !NO"33E:FE&;#//R.#(#=#=#? $)$?$?$A!#/4E4QK$ /1BCC)+<I I #,<<i %&7&G&GSS, 1 A A I I 0 @ @ H H /1BCC)+<I I )-==,<< RS55579JKKKK **,<<DDKKQQ  EMM) "MM	&&(88@@CC*Id D  &&(88@@OOPTUMN5668:KLLLL ++-==EELLRR  EMM) "MM	'')99AADD*Id E  '')99AAPPQUV ng..~'>??DK44A/3N,+11N ng.. >#:;;@GN00=+/( qAJJL  	"%( '..~/J/JER'//0L0LeTa!!&..}/I/I1MAN'//0K0KQOAOA a!! m & !((0O f  ##]^_]`a s1   $r C1r4  
r1
r,,r14
s">ss"c                    [         R                  R                  SS5      S:X  a  [        R                  " U US9$ [        U [        5      (       d   [        U [        5      (       d  [        S5      eU R                  R                  nUS:X  a  [        nOUS:X  a  [        nOwUS:X  a  [        nOjUS	:X  a  [        nO]US
:X  a  [        nOPUS:X  a  [        nOCUS:X  a  [        nO6US:X  a  [        nO)US:X  a  [        nOUS:X  a  [        nO[        SU S35      e[        U USS9n U R                  R!                  5        H  nM     SSKJn  UR.                  R0                  S:w  a  [3        S5      eU R                  nUR5                  5        HN  u  p[7        U	S5      (       d  M  U	R8                  b  M'  U	R:                  n
0 U
S'   U	R<                  " S/0 U
D6n	XU'   MP     SnSnSn[7        U S5      (       a  U R>                  S   OU R@                  nU R                  U   RB                  nU R                  U   RD                  nUS:X  a  [F        RH                  " USS9OUnUS:X  Ga  US:X  Ga  [K        U RL                  RL                  RN                  5       GH  u  nnUS:w  Ga}  URP                  nURR                  nURT                  nURV                  n[7        US5      (       Ga%  [7        US5      (       Ga  [7        US5      (       Ga  [Y        USU5      RD                  c  [Y        USU5      RD                  c  [Y        USU5      RD                  c  [)        [Y        US/ 5      =(       d    / 5      S:X  a  [)        [Y        US/ 5      =(       d    / 5      S:X  as  [)        [Y        US/ 5      =(       d    / 5      S:X  aP  [7        US 5      (       a  [Z        R\                  " UU5      Ul/        O[Z        R\                  " UU5      Ul0        US!-  nO[b        Rd                  " S"5        URf                  Rh                  nURf                  Rj                  nURf                  Rl                  n[7        US5      (       a  [7        US5      (       a  [7        US5      (       a  [Y        USU5      RD                  c  [Y        USU5      RD                  c  [Y        USU5      RD                  c  [)        [Y        US/ 5      =(       d    / 5      S:X  aa  [)        [Y        US/ 5      =(       d    / 5      S:X  a>  [)        [Y        US/ 5      =(       d    / 5      S:X  a  [n        URf                  l8        US!-  nO"US:X  a  US!-  nO[b        Rd                  " S#5        URf                  Rr                  n[7        US5      (       aX  [Y        USU5      RD                  c@  [)        [Y        US/ 5      =(       d    / 5      S:X  a  [t        URf                  l;        US!-  nGMm  [b        Rd                  " S$5        GM     [b        Rd                  " S%[x         S&[)        U RL                  RL                  RN                  5       S'U S(U S)U S*35        [{        U 5        U R|                  nU n[7        US+5      (       a&  UUl>        URL                  n[7        US+5      (       a  M&  UUl>        U R                  5        H
  nUUl>        M     U n[7        US+5      (       aA  [7        US,5      (       a  S-UR                  lA        URL                  n[7        US+5      (       a  MA  [7        US,5      (       a  S-UR                  lA        [        S.5       H"  n[        R                  " 5         [        5         M$     [        U 5        [F        RH                  " [        R                  U 5      U lH        [F        RH                  " [        R                  U 5      U lI        U n[7        US+5      (       as  [F        RH                  " [        R                  U5      UlH        [F        RH                  " [        R                  U5      UlI        URL                  n[7        US+5      (       a  Ms  U $ )0Nr  r  r  )r  ra  z:Unsloth: Your model needs to call `.get_peft_model` first!r  mistralqwen2r8  r9  r:  r;  qwen3r<  qwen3moer  z is not yet implemented!T)ra  rK  Fr   r  r  zBUnsloth: Unsuccessfully patched Trainer! Please file a bug report!rd  active_adaptersr   rS  lora_A
base_layerlora_magnitude_vector_unsloth_forwardr   zNot an error, but Unsloth cannot patch MLP layers with our manual autograd engine since either LoRA adapters
are not enabled or a bias term (like in Qwen) is used.zNot an error, but Unsloth cannot patch Attention layers with our manual autograd engine since either LoRA adapters
are not enabled or a bias term (like in Qwen) is used.zNot an error, but Unsloth cannot patch O projection layer with our manual autograd engine since either LoRA adapters
are not enabled or a bias term (like in Qwen) is used.zUnsloth z	 patched z layers with z QKV layers, z O layers and z MLP layers.r  r  r  r   r%  )Jr  r  r   r(   r  rn  r9   r:   r}  r{   rZ  apply_lora_mlp_swigluapply_lora_mlp_geglu_approxr  prepare_model_for_kbit_trainingrh  keysbase_model_name_or_pathr[  r  r   r	  r  r  r   r  itemsr\   rd  __dict__r   r  active_adapterr]  r^  r  r  rm  r  rk  r)  r   r   r   r   r  r   r  r  r$   r   r'  r@   rA   rB   apply_lora_qkvr  rL   apply_lora_or  r
   r;   r7  r  r  r  r  r  r  r  r  rv  r{  rw  ) r  ra  rZ  apply_lora_mlpr  r  r  all_configskeycurrent_confignew_argsn_mlpn_qkvn_or]  r^  _apply_lora_mlpr  r@  
mlp_moduler   r   r   r@   rA   rB   rL   r7  rL  rR   r   rK  s                                    rH   r  FastLlamaModel.patch_peft_modelH  s   
 ::>>1373> 11-G  %!566z5@
 @
 L 
 \\,,
 2N9$2N7"2N7"8N8#8N8#2N9$2N7"2N;&2N:%2N%	*=U&VWW/)C 
 $//446N 7 	1''004OOT  ''#.#4#4#6C77"//7)22+-(!/!9!9!EH!E#1C  $7  u/00 !!!$%% 	 ((8EE  055
 X% n> 	 1'(9(9(@(@A
U,
 "'J * 4 4I(00G * 4 4I  	844#GX66#Ix88$YiHMMU$WlGDIIQ$YiHMMU	3JB O USUV !  1H" M SQST !  	3JB O USUV !
 #:/ABB:?:J:J /;J7 271A1A /2J. 
++U //////FH--1111 v>CCK v>CCK v>CCKWV-DbIORPTUUWV-DbIORPTUUWV-DbIORPTUU0>EOO-QJE!W,
++U //FH-- v>CCKWV-DbIORPTUU.:EOO+1HC''Qi Br 	{m9S1B1B1I1I-J,K=g]3%~eWLJ	
 	u% -- ng..,:N)+11N ng.. )7%mmoF$2F! & ng..~'>??DK44A+11N ng.. >#:;;@GN00= qAJJL  	"%( '..~/J/JER'//0L0LeTa!!&..}/I/I1MAN'//0K0KQOAOA a!! rJ   c                 .   [        U S5      (       d  [        S5      eS nU n[        US5      (       a'  U" U5        UR                  n[        US5      (       a  M'  U" U5        U R                  5         U R	                  5        H  n[        US5      (       d  M  SUl        M     [        U S5      (       a(  U R                  5       n[        US5      (       a  SUl        [        U S	5      (       a(  U R                  5       n[        US5      (       a  SUl        U $ )
Nrq   zLUnsloth: I think you're passing a tokenizer, not the model to for_inference!c                     [        U S5      (       a  SU l        [        U S5      (       a  SU l        [        U S5      (       a  SU R                  l        SU l        g )Nra  Fr_  r  r  Tr\   ra  r_  r  r  r$  )rK  s    rH   _for_inference4FastLlamaModel.for_inference.<locals>._for_inference?  sP    q233+0(q*%%"
q1227=''4%)A"rJ   r  ra  Fr|  r_  r  )	r\   r}  r  r  r  ra  r|  r_  r  )r  r  rK  rR   
embeddingss        rH   rw  FastLlamaModel.for_inference8  s    ul++^ 		* a!!1A a!! 	q

 mmoFv78805- &
 5011335Jz:..&+
#5122446Jz:..&+
#rJ   c                   ^ [        U S5      (       d  [        S5      eU R                  5        H  n[        US5      (       d  M  U?M     U4S jnU n[        US5      (       a'  U" U5        UR                  n[        US5      (       a  M'  U" U5        U R                  5         U R                  5        H  n[        US5      (       d  M  TUl        M     [        U S5      (       a(  U R                  5       n[        US5      (       a  S	Ul	        [        U S
5      (       a(  U R                  5       n[        US5      (       a  S	Ul	        U $ )Nrq   zKUnsloth: I think you're passing a tokenizer, not the model to for_training!
_fast_lorac                    > [        U S5      (       a  TU l        [        U S5      (       a  SU l        [        U S5      (       a  SU R                  l        [        U S5      (       a  U ?g g )Nra  r_  Tr  r  r$  r  )rK  ra  s    rH   _for_training2FastLlamaModel.for_training.<locals>._for_trainingm  sb    q233+E(q*%%!
q1227>''4q011* 2rJ   r  ra  r|  r_  Tr  )r\   r}  rq   r  r  trainr  ra  r|  r_  r  )r  ra  r  r  rK  rR   r  s    `     rH   r{  FastLlamaModel.for_traininga  s%   ul++] 
 %%'Eul++$ (
	+ a!!!A a!! 	a mmoFv7880J- &
 5011335Jz:..&*
#5122446Jz:..&*
#rJ   r%  )zunsloth/llama-3-8b-bnb-4bitNNTN
sequentialNTNNFNFg      ?Fr  r  FFNN)rT  )T)r   r"  r#  r$  staticmethodr  r  r  r  r8   r  rw  r{  r%  r%  rJ   rH   rv  rv  %  s(     $ $L 2!!!$!$+i  i V      
 "%. ?#5@ @D  &/m m^ & &P , ,rJ   rv  )PatchFastRL)FastLanguageModel)r   r?   )FN)NNNN)NNN)NNNNFFNN)
NNNNNNNNNN)NNNNNNNNNr   r   )r_   r  rD  r  typingr   r   r   r   _utilsr	   r
   r   r   r   r   utils.packingr   r   utils.attention_dispatchr   r   r   r   torch.nn.functionalr   r  rr   unsloth_zoo.utilsr   r   unsloth_zoo.hf_utilsr   r   r   unsloth_zoo.peft_utilsr   rS   r   r   r   r    r!   r"   rj  transformers.modeling_layersr#   r  r  r$   r%   r&   rg  r'   kernelstokenizer_utilsvisionr(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   &transformers.models.auto.modeling_autor4   r5   r{  r  r6   r7   r8   r  r9   r:   saver;   r  r  rn   sysr  huggingface_hub.utilsr<   huggingface_hub.utils._tokentritonr  r  r  r  r  r=   empty_cacher  current_devicer  r  rI   rN   rO   r   r   r#  
functionalsoftmaxr   r/  r   peft.utils.otherrQ   r  rf   r   r   matmulr   r  r   silur   r   r   r   r   r   r   r   compiletorch_compile_optionsr  r  r   r"  r  r/  ra   r`   r0  __DTYPE_MAPr  r  r  r  _disable_dynamor  Moduler  r)  r1  rR  r~  rv  rlr  r%  rJ   rH   <module>r     s    	   / /  > 2 "  = < 1 
 =  34 ,wx/@@ ,G 
   ! *  O : H H I ) ! ! ! 7/ 1t#2>H..D  %ii++O11jj,,O22
 # #hh1199 ;CCC 1  01 0101 01 
	01n SlS || K<<K U5<<01K\ !88..33  Z^8 ||ZZ
(" 5D<QR
 S
" 6:-1/348#/3GKZ5<<Z5 12Z5 U\\*	Z5
 5++,Z5 U5<<01Z5 Z5 Z5 5++,Z5 "%ell(B"CDZ5 5<<%,,/%:M1NNOZ5B -1/348(- %/3GKQ<<Q U\\*	Q
 5++,Q U5<<01Q  ~Q ~Q 5++,Q "%ell(B"CDQ 5huU->->@Q@Q-Q'RSSTQl u}}	MM5==u}}	MM5==	NNENN -159-1/39=15 $(,+/"&O(()O 12O U\\*	O
 5++,O d5#4#456O E--.O ~O  ~O #4.O $O 5))*Oh (M!6o4f %G$H !R"j  (
 (
`i588?? iX-(< -d}S588?? }S@behhoo bJBJi iX+   /kj,!%d8*')&766s$   V/ V?  W /
V<?WW