
    oi3                         / S Qr SSKrSSKrSSKrSSKJrJrJrJrJ	r	J
r
JrJrJrJr  SSKJr  SSKr/ SQrSSSSS/ SQSS	// S
Q/ SQ4	S\S\S\S\S\\   S\\   S\\   S\\   S\\   S\4S jjr S r S rg))get_peft_regexmerge_and_overwrite_loramerge_and_dequantize_loraSKIP_QUANTIZATION_MODULESget_lora_layer_modules(requires_grad_for_gradient_checkpointing    N)
AnyCallableDictListOptionalSetTupleTypeTypeVarUnion)OrderedDict)lm_headmulti_modal_projectormergermodality_projectionrouterzmlp.gatezblock_sparse_moe.gatemambaT)visionimagevisualpatchlanguagetext)	self_attn	attentionattn)mlpfeed_forwardffndensefinetune_vision_layersfinetune_language_layersfinetune_attention_modulesfinetune_mlp_modulestarget_modulesvision_tagslanguage_tagsattention_tagsmlp_tagsreturnc
                 n  ^ U(       d  U(       d  [        S5      eU(       d  U(       d  [        S5      e SSKJn
  U R                  5       nU VVs/ s H2  u  p[	        U[
        R                  R                  5      (       d  M0  UPM4     nnnU
" S U 5       5      nUcI  / n0 n[        UR                  5       5       H'  u  nu  nnUS:w  a  UR                  U5        M"  UUU'   M)     O[        U5      [        L d   e[        U5      n / nU(       a  UU-  nU(       a  UU-  n/ nU(       a  UU-  nU(       a  UU	-  nSR                  U5      nSR                  U5      nSSR                  S	 U 5       5      -   S
-   nSU-   S-   U-   S-   U-   S-   mU(       a  ST-   S-   U-   S-   U-   S-   m [        U4S jU 5       5      nU(       d  SU-   S-   U-   S-   m [        U4S jU 5       5      nU(       d  Ub  [        SU S35      eU(       d#  [        SU R                  R                    S35      e T$ s  snnf )zH
Create a regex pattern to apply LoRA to only select layers of a model.
zaUnsloth: No layers to finetune - please select to finetune the vision and/or the language layers!zaUnsloth: No modules to finetune - please select to finetune the attention and/or the mlp modules!r   )Counterc              3   H   #    U  H  oR                  S 5      S   v   M     g7f).N)rsplit.0xs     P/home/james-whalen/.local/lib/python3.13/site-packages/unsloth_zoo/peft_utils.py	<genexpr>!get_peft_regex.<locals>.<genexpr>K   s      KNq#r!2Ns    "   |z(?:c              3   N   #    U  H  n[         R                  " U5      v   M     g 7f)N)reescaper7   s     r:   r;   r<   g   s     ,WCVaRYYq\\CVs   #%)z.*?(?:z).*?(?:z).*?z.*?z#)|(?:\bmodel\.layers\.[\d]{1,}\.(?:z)\.(?:z))c              3   l   >#    U  H)  n[         R                  " TU[         R                  S 9v   M+     g7f)flagsNr@   searchDOTALLr8   nameregex_matchers     r:   r;   r<   u   "     ]nd		-ryyAn   14c              3   l   >#    U  H)  n[         R                  " TU[         R                  S 9v   M+     g7frD   rF   rI   s     r:   r;   r<   }   rL   rM   zKUnsloth: No layers to finetune? You most likely specified target_modules = z incorrectly!z#Unsloth: No layers to finetune for z. Please file a bug report!)RuntimeErrorcollectionsr2   named_modules
isinstancetorchnnLinear	enumerateitemsappendtypelistjoinanyconfig_name_or_path)modelr'   r(   r)   r*   r+   r,   r-   r.   r/   r2   modulesrJ   modulelinear_modulesall_linear_modulesonly_linear_modulesprojection_modulesjprojcountregex_model_partsregex_componentsmatch_linear_modulescheckrK   s                            @r:   r   r   -   s     "*Bo
 	
 &.Bo
 	
 	#!!#G/6^w|t*VUXX__:]dwN^  KN KK    )*<*B*B*D EA}ez#**40+,"4(	 !F
 	N#t+,+">2 #4#C#4#4#E#4!#3#F#3#3#@#3!23!12!CHH,WCV,W$WWZ^^&&	%	&	)	*,1	2   ../1AB()+01 	 ]n]]E((,-/45 	 	 ]n]]E^/YZhYiivw
 	
 1%,,2L2L1MMhi
 	
 	{ _s   /H1H1c            	      
   SS K n [        R                  R                  U R                  R
                  R                  5      S   n[        R                  " U5      n/ nU GH  nUS:X  d  UR                  S5      (       d  M"  SUS [        S5      *   3n[        SU 3[        5       [        5       5        [        [        U5      5      nU Vs/ s H2  owR                  S5      (       d  UR                  S5      (       d  M0  UPM4     nn[        U5      S:X  a  M  [        SU SS	R!                  U5       S
3[        5       [        5       5        X6 Vs/ s H  n[        U5      XW4PM     sn-  nGM      [#        U5      $ s  snf s  snf )Nr   z__init__.pyz.pyzpeft.tuners.lora.zimport rU   zfrom z	 import (z, rB   )peft.tuners.loraospathsplittunerslora__file__listdirendswithlenexeclocalsglobalsdireval
startswithr[   tuple)peftrp   filesLinear_LoRA_Layersfileitemr`   r9   s           r:   r   r      sB   77==))223A6DJJtE= e(<(<h"4#e*#5"67wtfvx3d4j/%Xgh)?)?1::hCW1gXw<1huTF)DIIg$6#7q968WYOGDGqQ2GDD  	#$$ Y Es   /E;>E;F c                    S n U R                  5        H)  u  p#[        UR                  5      S:w  a  U" SSUS5        M+      S n S n S nU R                  5        H  u  p&UR                  (       d  M    O   Uc  g [
        R                  " SS	W5      nUR                  S
5      n[        U5      S:X  a  [        S5      eS n[        [        U5      S-
  SS5       H  n	Xy   n
SS
R                  US U	 5      -   n[
        R                  " SU5      (       a  M<  [        U5      n[        US5      (       aw  [        R                  " UR                   5      nSU
 S3U;   a  U	S-   n  OJ[
        R                  " SSU
5      nSU S3U;   a  U	n  O$[
        R                  " SU-   U5      b  U	n  O M      Uc  U" SSWS5        UR#                  U5        g  SS
R                  US U 5      -   n[        U5      n[        US5      (       aV  UR$                  R&                  R(                  S;   a2  U nU R                  5        H  u  p[        US5      (       d    OUnM     Un [+        SU S35        Sn[        US5      (       a/   UR-                  5       nU" SSUS5        UR#                  U5        Sn U(       a  U" SSUS 5        UR/                  U5        g !    GM  = f!   Sn N4= f)!Nc                 .   [        SU 35      n/ nUR                  5        HF  n[        USS5      n[        USS5      nX;   d  X;   a  M)  X;   d  X;   a  M5  UR                  U5        MH      [	        SU S35        U H  n	[	        SUS S  S35        M     g )	Nzmodule.__qualname__ __name__z = OrderedDict()zmodule.registerr5   z(hook))r|   valuesgetattrrX   rx   )
name1name2ra   _hooks	old_hooksother_hooksvaluequalnamerJ   hooks
             r:   register_other_hooksFrequires_grad_for_gradient_checkpointing.<locals>.register_other_hooks   s    76(+,	%%'Eunb9Huj"5D E$5t%-$$U+ ( 	wvh./0D?6#2;-v67      r   enable_input_require_gradsmake_inputs_require_grad_forward_hooksc                    [        U5      nU[        R                  L a  UR                  S5        g  [	        US5      (       a)  UR
                  b  UR
                  R                  S5        g [	        US5      (       a)  UR                  b  UR                  R                  S5        g [        S5      e! [         a  n[        SU 35      eS nAff = f)NTlosslogitsz9Neither loss nor logits are available for grad post hook.z2Unsloth: Failed to make output require gradients: )
rY   rS   Tensorrequires_grad_hasattrr   r   
ValueError	ExceptionrO   )ra   inputoutputtype_outputes        r:   requires_grad_post_hookIrequires_grad_for_gradient_checkpointing.<locals>.requires_grad_post_hook   s    6l%,,&!!$']66**v{{/FKK..t4VX..6==3LMM006$%`aa ]"%WXYWZ#[\\]s#   9B1 ,9B1 &B1 1
C;C		Cc                 8   [        U5      nU[        R                  L a  UR                  S5        g U[        L d	  U[
        L aN  [        U5      S:X  a  [        S5      e[        R                  " US   5      (       a  US   R                  S5        g g [        S5      e)NTr   z0Unsloth: Failed to make input require gradients!)	rY   rS   r   r   r~   rZ   rw   rO   is_floating_point)ra   r   
type_inputs      r:   requires_grad_pre_hookHrequires_grad_for_gradient_checkpointing.<locals>.requires_grad_pre_hook   s    %[
%  &5 J$$65zQ"#UVV &&uQx00a''- 1 QRRr   z\.([\d]{1,})\.z[\1].r4   zUnsloth: Model has 0 layers?r=   r5   zmodel.z\[[\d]{1,}\]forwardzself.(r   zin self.:zfor [^\s]{3,} in self\.r   r]   )CLIPVisionConfigSiglipVisionConfigget_input_embeddingszUnsloth: Making `z` require gradientsTFr   _forward_pre_hooks)rQ   rw   r   named_parametersrequires_gradr@   subrq   rO   ranger[   rG   r|   r   inspect	getsourcer   register_forward_hookr]   	__class__r   printr   register_forward_pre_hook)r_   r   rJ   ra   r   r   paramname_componentsfinal_whererf   	name_currname_prer   module_listmodule_name
old_modulestill_need_patchings                    r:   r   r      s-    	 ++-v$$%* ,* 	 	 . 	] 	S 	 E--/ 0}f66#Xt4DjjoO
?q 9::K3')1b1#&	sxx(;<<99_h//h69%%",,V^^< yk#w.!e &&"i@K+a(G35CWMY3 24 	 	%%		
 	$$%<=SXXol{&CDDK+Fvx  fmm&=&=&F&FJu&u
#(#6#6#8K6#9::EJ $9 	k]*=
>?v-..	'002F )) 	 (()@A"' 	$$ 		
 	(()?@W Hz	'"&s   3 K=.K KK)__all__r   rS   ro   typingr	   r
   r   r   r   r   r   r   r   r   rP   r   r@   r   boolstrr   r   r    r   r:   <module>r      s   "   	 X X X # 		  )-(,(,(,-1-T.8&-B-P-UZ!%Z "&Z "&	Z
 "&Z "&cZ "&cZ "&cZ "&cZ "&cZ 	Zv %$ b	F r   