
    oi=                     Z   S SK r S SKrS SKrS SKrS SKJrJr  S SKJr  S SK	J
r
  S SKrS SKrS SKJr  SSKJr  S SKJrJrJrJr  S S	KJr  S S
KJr  S SKJr  S SKJr  S SKr/ SQr\ R>                  " \ 5      r!\RD                  RG                  SS5      RI                  5       RK                  5       S;   r&SS1r'S\(4S jr)S\(4S jr*S r+Sr,S\-S\(4S jr.S SK/J0r1J2r2  \" \15      \" S5      :  a  S rOS r S SKJ3r4   " S  S!\45      r5 S(S" jr6 " S# S$\5      r7S% r8S& r9S' r:g!   S SK/J4r4   N0= f))    N)	dataclassfield)Optional)wraps)
SFTTrainer   )is_bfloat16_supported)configure_padding_freeconfigure_sample_packingenable_padding_free_metadataenable_sample_packing)unsloth_train)UnslothVisionDataCollator)get_transformers_model_type)Version)UnslothTrainingArgumentsUnslothTrainerr   _patch_trl_trainerr   !UNSLOTH_DISABLE_AUTO_PADDING_FREE >   1onyestruegemma2gpt_ossreturnc                 R    U b  [        U SS5      (       d  g[        U SS5      (       + $ )NpackingF_unsloth_disable_auto_packing)getattrconfigs    I/home/james-whalen/.local/lib/python3.13/site-packages/unsloth/trainer.py_should_packr%   ?   s+    ~WVY>>v>FFF    c                 h    U b  [         (       d  [        U SS5      (       a  g[        U SS5      (       + $ )Nr   Fpadding_free)_AUTO_PADDING_FREE_ENV_DISABLEDr!   r"   s    r$   _should_auto_padding_freer*   E   s1    **69e,,v~u555r&   c                     U c  g S H#  u  p[        X5      (       d  M  [        XU5        M%     [        U S5      (       a  [        U SS5        [        U SS5        g )N))r   F)r(   Fremove_unused_columnsTr    )hasattrsetattr)r#   attrvalues      r$   _disable_sample_packingr1   O   sV    ~D6  F%( E v.///6F3T:r&   )zpacking is not supportedzpadding-free trainingzpassing a custom data collatorexcc                 h   ^ [        U 5      R                  5       m[        U4S j[         5       5      $ )Nc              3   ,   >#    U  H	  oT;   v   M     g 7fN ).0msgmessages     r$   	<genexpr>2_should_skip_auto_packing_error.<locals>.<genexpr>c   s     B)A#g~)As   )strlowerany_AUTO_PACK_SKIP_MESSAGES)r2   r9   s    @r$   _should_skip_auto_packing_errorr@   a   s%    #hnnGB)ABBBr&   )__version__ProcessorMixinz4.45.2c                 &    U R                   " U0 UD6$ r5   )traintrainerargskwargss      r$   r   r   k   s    }}d-f--r&   c                     [        U5      S:w  d  [        U5      S:w  a  [        S5      e[        S5        [        U 5      $ )Nr   a  Unsloth: Our custom gradient accumulation fixed trainer does not support other arguments.
If you want to use our fix inside of HF, please update `transformers` to the latest version via:
`pip uninstall transformers -y && pip install --upgrade --no-cache-dir transformers`a  Unsloth: Using our custom gradient accumulation fixed trainer, which is not feature complete.
If you want to use our fix inside of HF, please update `transformers` to the latest version via:
`pip uninstall transformers -y && pip install --upgrade --no-cache-dir transformers`)lenRuntimeErrorprint_unsloth_trainrE   s      r$   r   r   p   sH    t9>S[A-g 
 	c	

 g&&r&   )	SFTConfigTrainingArgumentsc                   4   ^  \ rS rSrSS\4U 4S jjjrSrU =r$ )r      embedding_learning_ratec                 *   > Un[         TU ]  " U0 UD6  g r5   )super__init__)selfrS   rG   rH   	__class__s       r$   rV   !UnslothTrainingArguments.__init__   s    "9$)&)r&   r6   r5   )__name__
__module____qualname____firstlineno__floatrV   __static_attributes____classcell__rX   s   @r$   r   r      s    * * *r&   r   c           
         US   nUR                  SS5      n0 0 S.nU R                  5        Hz  u  pxUR                  (       d  M  UR                  S5      (       aE  US [	        S5      *  n	XR                  S5      S-   S  n	[        S	US
 SUS
 SU	 S35        XS   U'   Ms  XS   U'   M|     [        US   R                  5       5      UUS.[        US   R                  5       5      UUS./n
U" U
40 UD6nU$ )Nlrweight_decayg        )non_embeddings
embeddingszmodules_to_save.default.weightz.modules_to_save.default.weight.r   zUnsloth: Setting lr = z.2ez instead of z for rf   re   )paramsrd   rc   )	getnamed_parametersrequires_gradendswithrJ   rfindrL   listvalues)modeloptimizer_clsoptimizer_kwargsembedding_lrrc   rd   param_groupsnameparampartial_nameoptimizer_grouped_parameters	optimizers               r$   _create_unsloth_optimizerrz      s@    
$	B#''<L L
 --/""==9:: I3'H#I"IJL'(:(:3(?!(C(EFL(c(:,r#heT`Saabc 05&t,38)*40 0 <(89@@BC(	
 <5<<>?(	
$  :O>NOIr&   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )r      c                   > [        U R                  SS 5      nUc  [        TU ]  5       $ U R                  c?  [
        R                  " U R                  5      u  p#[        U R                  UUU5      U l        U R                  $ )NrS   )	r!   rG   rU   create_optimizerry   r   get_optimizer_cls_and_kwargsrz   rp   )rW   rS   rq   rr   rX   s       r$   r~   UnslothTrainer.create_optimizer   sz    ")$))5NPT"U"*7+-->>!.8.U.U		/+M 7

 '	DN ~~r&   )ry   )rZ   r[   r\   r]   r~   r_   r`   ra   s   @r$   r   r      s     r&   r   c                 L   ^^ U R                   m[        T5      UU4S j5       nU$ )Nc                 :  > [        [        R                  " T5      R                  R	                  5       5      nSU;   a  SU;   a  UR                  S5      US'   SU;   Ga  [        [        5      [        S5      :  Ga  UR                  SS 5      nUR                  S5        UR                  S5        [        R                  " T5       Vs0 s H#  nUR                  (       d  M  UR                  U_M%     nnU Vs0 s H!  n[        XG5      (       d  M  U[        XG5      _M#     nnSSKJn	  [        [        R                  " T5      R                  R	                  5       5      [        [        R                  " U	5      R                  R	                  5       5      -
  n
0 n0 nUR#                  5        H$  u  pX;   a  XU'   M  X;   d  X;   a  XU'   M   XU'   M&     UR%                  U5        ['        XI5      (       d	  T" S0 UD6nOUnUnXS'   T" U /UQ70 UD6  g s  snf s  snf )	Nprocessing_class	tokenizerrG   z0.13.0.dev0rW   r   rO   r6   )setinspect	signature
parameterskeyspopr   trlremovedataclassesfieldsinitru   r-   r!   transformersrP   itemsupdate
isinstance)rW   rG   rH   trainer_paramstraining_argsr   config_fieldsru   config_dictrP   moved_paramstrainer_kwargsadditional_config_kwargskeyr0   r#   config_classoriginal_inits                   r$   new_init/_backwards_compatible_trainer.<locals>.new_init   s    W..}=HHMMOP/K64I)/K)@F%&f73<7=3I#I"JJvt4M !!&)!!&)
 )//==E:: "

E!=   *)D=/ 3gm22)   7w00>IINNPQTW!!"34??DDFU L
  N')$$lln
(*/3'(C,@49S149S1 - 78 m??%44& $F#6Nd,T,V,as   	H"H9HH)rV   r   )trainer_classr   r   r   s    ` @r$   _backwards_compatible_trainerr      s-    !**M
=?- ?-B Or&   c                    ^ [        U SS 5      nUc  g [        USS5      (       a  g UR                  m[        T5      U4S j5       nX!l        SUl        g )Nr   _unsloth_auto_packing_wrappedFc                   > S n[        U5      S:  a  US   nOUR                  S5      nUR                  S5      nSnSnUbf  [        USS 5      nUbV  [        U5      n[	        S U 5       5      n[        USS 5      n	U	c  / n	[	        S	 U	 5       5      nU=(       d    [        US
5      nUR                  S5      =(       d    UR                  S5      n
UR                  S5      nUS L=(       dL    [        U
[        5      =(       d5    U=(       d,    U=(       d#    [        R                  R                  SS5      S:H  n[        [        USS5      5      nU(       a<  [        US5      (       a  [        USS5        [        US5      (       a  [        USS5        U(       aZ  U(       aS  SnUc  [        U
[        5      (       a  SnO%U(       a  SnOU(       a  SSR                  W5       3nSU S3n[        U5        Sn[        U5      (       a)  U(       d"  [        U5        Sn[         R#                  S5        Sn[        USS 5      SL nU(       dE  U(       a  [%        U5        O2['        U5      (       a"  [%        U5        Sn[         R#                  S5         T" U /UQ70 UD6  [        U SS 5      n[        U=(       a    [        USS5      5      n[        U=(       a    [        USS5      5      nU(       a  Ub  [        USS5        [        USS5        U(       d@  U(       a9  U(       d  [        U5      (       a"  [/        U R0                  U 5        [        S5        g U(       d5  U(       a-  [3        U R0                  U 5        U(       a  SOS n[        U5        g g g ! [(         aR  nU(       aE  [+        U5      (       a5  [         R#                  SU5        [-        U5        SnT" U /UQ70 UD6   S nAGNIe S nAff = f)!N   r   rG   rp   Fr#   c              3   2   #    U  H  o[         ;   v   M     g 7fr5   )PADDING_FREE_BLOCKLISTr7   xs     r$   r:   D_patch_sft_trainer_auto_packing.<locals>.new_init.<locals>.<genexpr>-  s      +9DA//s   architecturesc              3   B   #    U  H  oR                  S 5      v   M     g7f)ForConditionalGenerationN)rl   r   s     r$   r:   r   5  s      DQqJJ9::Ms   vision_configr   r   data_collatorUNSLOTH_RETURN_LOGITS0r   r   r(   zcustom data collatorzprocessor-based modelzvision-language modelzunsupported model type(s): z, z!Unsloth: Sample packing skipped (z detected).Tz8Unsloth: Sample packing enabled for SFTTrainer instance.zDUnsloth: Padding-free batching auto-enabled for SFTTrainer instance.zXUnsloth: Auto sample packing failed because trainer reported an incompatible setup (%s).uJ   🦥 Unsloth: Packing enabled - training is >2x faster and uses less VRAM!uB   🦥 Unsloth: Padding-free auto-enabled, enabling faster training.u=   🦥 Unsloth: Padding-free enabled, enabling faster training.)rJ   ri   r!   r   r>   r-   r   rB   osenvironboolr.   joinrL   r%   r   loggerinfor
   r*   
ValueErrorr@   r1   r   rp   r   )rW   rG   rH   
config_argrp   is_unsupported_modelis_vlmmodel_configmodel_typesr   r   r   blockedrequested_packreasonr9   packing_activeauto_padding_free_activepadding_free_requestedr2   trainer_argstrainer_packingtrainer_padding_freer   s                          r$   r   1_patch_sft_trainer_auto_packing.<locals>.new_init  s   
t9>aJF+J 

7#$"5(D9L'9,G'* +9D+ ($
 !(ot L ($&M DQ   I7<#I!::&89TVZZ=T

?3 $& *N; $
 

6<C 	 gj)UCDz9--
Iu5z>22
NE:~+F$4Dn)U)U00%6tyy7M6NO<VHKPG'N
##G$Z0!NKKRS#( !(^T!Jd!R%&z2*:66&z2+/(Z	$000 tVT2|WiQV0WX#IW\>5I 
 |/L)U3L.%8 <#=#=!$**d3\ 1(T: , UT 
 'N 2A  
	"A#"F"Fn (
3!&d4T4V44
	s   N 
O'AO"!O""O'T)r!   rV   r   r   )
trl_modulesft_trainerr   r   s      @r$   _patch_sft_trainer_auto_packingr     sb    *lD9K{;UCC((M
=w wr $04K-r&   c            
      z   SS K n [        U S5      (       a  g [        U 5      [        S5      ::  a  g SS Kn [	        U R
                  5      n[        S U 5       5      n[        S U 5       5      n[        X#-  5      nU H"  n [        SU SU SU S	3[        5       5        M$     [        U 5        S
U l        g !    M>  = f)Nr    __UNSLOTH_BACKWARDS_COMPATIBLE__z0.11.0c              3   n   #    U  H+  oR                  S 5      (       d  M  US[        S 5      *  v   M-     g7f)TrainerNrl   rJ   r   s     r$   r:   %_patch_trl_trainer.<locals>.<genexpr>  s-      &1ZZ	5J
S^Ok   55c              3   n   #    U  H+  oR                  S 5      (       d  M  US[        S 5      *  v   M-     g7f)ConfigNr   r   s     r$   r:   r     s*     W;a**XBV)a(3x=.);r   ztrl.z5Trainer.__init__ = _backwards_compatible_trainer(trl.zTrainer, trl.zConfig)T)r   r-   r   trl.trainerdirrF   r   rn   execglobalsr   r   )r   trl_classestrl_trainerstrl_configsr   s        r$   r   r     s    s677s|wx((ckk"K &1 L W;WWK|12K	qcNqcQ^_`^aahi	  $C(+/C(	s    B55B:)g-C6
?);loggingr   psutilwarningsr   r   r   typingr   	functoolsr   r   r   r   r   r	   unsloth.utilsr
   r   r   r   unsloth_zoo.training_utilsr   rM   unsloth_zoo.vision_utilsr   unsloth_zoo.hf_utilsr   unsloth_zoo.utilsr   __all__	getLoggerrZ   r   r   ri   stripr=   r)   r   r   r%   r*   r1   r?   	Exceptionr@   r   rA   transformers_versionrB   rN   rP   r   rz   r   r   r   r   r6   r&   r$   <module>r      sG    	   (   
   #  = %  
		8	$"$**..'#%'%%'/#0 
  GD G6 6; C Ct C M
 78#44.
'/2
*0 * 	(VZ *EPD5N0s/.s   4D   D*