
    oiG2                    n   S SK Jr  S SKrS SKJrJrJr  S SKJr  S SK	J	r	  S SK
JrJrJrJrJrJr  S SKJrJrJr  S SKJrJrJrJrJr  \R6                  \R8                  \R:                  \R<                  \R>                  \R@                  4r!\RD                  RF                  \RH                  RF                  \RJ                  RF                  \RL                  RF                  \RN                  RF                  /r(\\RR                  \RT                  \RV                  \RX                  \RZ                  \R\                  4   r/\R`                  Rb                  \R`                  Rd                  \R`                  Rf                  \R`                  Rh                  \Rj                  \Rl                  4r7 " S	 S
\5      r8g)    )annotationsN)AnyOptionalUnion)nn)tqdm)adaloralohalokrloraoftshira)	BaseTunerBaseTunerLayer_delete_auxiliary_adapter)2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGModulesToSaveWrapperPeftType_get_submodulesget_auto_gptq_quant_linearc                     ^  \ rS rSrSrSU 4S jjrSU 4S jjr        SS jrSS jrSS jr	\
S 5       rSSS	 jjr\
S
 5       r    S     SS jjrSS jrSS jrSS jrSrU =r$ )
MixedModel7   a  
A class that allows to mix different types of adapters in a single model.

Note: This class should usually not be initialized directly. Instead, use `get_peft_model` with the argument
`mixed=True`.

Args:
    model (:obj:`nn.Module`):
        The model to be tuned.
    config (:obj:`PeftConfig`):
        The config of the model to be tuned. The adapter type must be compatible.
    adapter_name (:obj:`str`):
        The name of the first adapter.
c                &   > [         TU ]  XU5        g N)super__init__)selfmodelconfigadapter_name	__class__s       Q/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/mixed/model.pyr   MixedModel.__init__G   s    5    c           	        > [        U[        R                  5      (       d5  [        U R                  R
                   S[         S[        U5       S35      e[        TU ]%  U5        g)z
A helper method to check the config when a new adapter is being added.

Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

z only supports z configs, but got .N)

isinstanceConfigs__args__
ValueErrorr"   __name__COMPATIBLE_TUNER_TYPEStyper   _check_new_adapter_config)r   r    r"   s     r#   r/   $MixedModel._check_new_adapter_configJ   sa     &'"2"233>>**+?;Q:RRdeijpeqdrrst  	)&1r%   c                p   [        U[        R                  5      (       a%  [        R                  R                  " X/UQ70 UD6  g [        U[
        R                  5      (       a%  [
        R                  R                  " X/UQ70 UD6  g [        U[        R                  5      (       a%  [        R                  R                  " X/UQ70 UD6  g [        U[        R                  5      (       a%  [        R                  R                  " X/UQ70 UD6  g [        U[        R                  5      (       a%  [        R                   R                  " X/UQ70 UD6  g [        U["        R$                  5      (       a%  ["        R&                  R                  " X/UQ70 UD6  g [)        S[+        U5       S[,         S35      e)NzUnsupported config type , should be one of r'   )r(   r	   AdaLoraConfigAdaLoraModel_create_and_replacer   
LoraConfig	LoraModelr
   
LoHaConfig	LoHaModelr   
LoKrConfig	LoKrModelr   	OFTConfigOFTModelr   ShiraConfig
ShiraModelr+   r.   r-   )r   r    argskwargss       r#   r5   MixedModel._create_and_replaceX   s=    fg3344  44TSDSFS00NN..tMdMfM00NN..tMdMfM00NN..tMdMfM..LL,,TKDKFK 1 12200OOO7V~EXYoXppqrssr%   c                  ^ [        XU5        [        US5      (       a  UR                  5       nO[        US5      (       a  UR                  n[        US5      (       d3  UR                  Ul        [        US5      (       a  UR
                  Ul        [        USS 5      bc  [        US5      (       a  UR                  UR                  l        OUR                  Ul        UR                  UR                  R                  5        UR                  5        Hv  u  mn[        U4S j[         5       5      (       a%  UR                  UR                  R                  5        ST;   d  MQ  UR                  UR                  R                  5        Mx     g )N
base_layerquant_linear_modulebiasstatec              3  ,   >#    U  H	  oT;   v   M     g 7fr    ).0prefixnames     r#   	<genexpr>-MixedModel._replace_module.<locals>.<genexpr>   s     9fT>   ranknum)setattrhasattrget_base_layerrE   weightrF   getattrrG   rD   todevicenamed_modulesanyPREFIXES)r   parent
child_name
new_modulechildmodulerL   s         @r#   _replace_moduleMixedModel._replace_modulem   s"   J/
 5,''((*EU122--Ez<00 %Juf%%"'**
5'4(4z<00.3kk
%%+#(;;
 MM%,,--. '446LD&9999		%,,--.D 		%,,--.	 7r%   c                @  ^ UR                  5        H-  u  mn[        U4S j[         5       5      (       a  M&  SUl        M/     U R                   H  n[        U R                  U   SS5      nUS:X  a  M%  US:X  a+  UR                  5        H  u  mnST;   d  M  SUl        M     MV  US:X  ac  UR                  5        HM  n[        U[        5      (       d  M  [        US5      (       d  M-  UR                  c  M<  SUR                  l        MO     M  [        SU S	35      e   g )
Nc              3  ,   >#    U  H	  oT;   v   M     g 7fr   rI   )rJ   rK   ns     r#   rM   >MixedModel._mark_only_adapters_as_trainable.<locals>.<genexpr>   s     :v{rO   FrF   noneallT	lora_onlyzRequested bias: z, is not implemented.)named_parametersrY   rZ   requires_gradactive_adaptersrU   peft_configmodulesr(   LayersrR   rF   r+   )r   r   pactive_adapterrF   mrd   s         @r#    _mark_only_adapters_as_trainable+MixedModel._mark_only_adapters_as_trainable   s    **,DAq::::"' - #22N4++N;VVLDv~u}!224DAq{*. 5 $A!!V,,F1C1CHZ/3, ) !#3D69N!OPP 3r%   c                   UR                  SS 5      n[        U5      nUc  Ub#  [        SU R                  R                   S35      eUR                  SS5      nUR                  SS5      nU(       d  U(       a#  [        SU R                  R                   S35      e[        U [        R                  5      (       a$  [        R                  R                  " XU40 UD6nU$ [        U [        R                  5      (       a$  [        R                  R                  " XU40 UD6nU$ [        U [        R                  5      (       a$  [        R                   R                  " XU40 UD6nU$ [        U ["        R$                  5      (       a$  ["        R&                  R                  " XU40 UD6nU$ [        U [(        R*                  5      (       a$  [(        R,                  R                  " XU40 UD6nU$ [        U [.        R0                  5      (       a$  [.        R2                  R                  " XU40 UD6nU$ [        S[5        U 5       S	[6         S
35      e)Ngptq_quantization_configz$GPTQ quantization not supported for  (yet).loaded_in_8bitFloaded_in_4bitz-8bit and 4bit quantization not supported for zUnknown config type r2   r'   )getr   r+   	peft_typevaluepopr(   r	   r3   r4   _create_new_moduler   r6   r7   r
   r8   r9   r   r:   r;   r   r<   r=   r   r>   r?   r.   r-   )	r    r!   targetrA   ru   AutoGPTQQuantLinearrw   rx   r]   s	            r#   r}   MixedModel._create_new_module   s   #)::.H$#O 89QR$06I6UCFDTDTDZDZC[[bcdd$4e<$4e<^LVM]M]McMcLddklmmfg3344 --@@W]haghJ  00::6QWb[abJ  00::6QWb[abJ  00::6QWb[abJ  ..88v`Y_`J
 	  1 122))<<VSYd]cdJ  3DL>ATUkTllmnoor%   c                *   U R                  XS9  U R                  R                  5        H`  n[        U[        5      (       d  M  UR
                  (       a&  [        R                  " S5        UR                  5         UR                  XS9  Mb     Xl
        g )N)inference_modezJAdapter cannot be set when the model is merged. Unmerging the model first.)set_auxiliary_adaptersr   rm   r(   rn   mergedwarningswarnunmergeset_adapterrp   )r   r!   r   r_   s       r#   r   MixedModel.set_adapter   sn    ##L#Pjj((*F&&))==MM"noNN$""<"O + +r%   c                    U R                   c2  US   [        ;  a  [        S5      e[        [        US      5      U l         U $ )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r+   set)rl   model_configs     r#   _prepare_adapter_config"MixedModel._prepare_adapter_config   sK    %%-L)1cc !STT),B<P\C]^*K& r%   c                F  ^^^ U(       a&  [        U R                  SS 5      S:X  a  [        S5      eUU4S jnU R                  R                  5        V^Vs/ s H(  u  mn[	        U4S j[
         5       5      (       a  M&  TPM*     nnnSU(       a  SOS-   S	-   n	[        X(       + U	S
9 H  m [        U R                  T5      u  pn[        US5      (       a2  U(       a  U" U5        U R                  XUR                  5       U5        M`  [        U[        5      (       d  Mw  UR                  UR                     n[        US5      (       a'  U(       a  UR!                  TTS9  UR                  5       n[#        XU5        M     U R                  $ s  snnf ! [         a     M  f = f)Nquantization_methodgptqz4Cannot merge layers when the model is gptq quantizedc                (  > / nU n[        US5      (       a0  UR                  U5        UR                  n[        US5      (       a  M0  [        US S USS  5       H&  u  p4UR	                  TTS9  UR                  Ul        M(     U R	                  TTS9  g )NrD      
safe_mergeadapter_names)rR   appendrD   zipmerge)r_   pathlayerlayer_beforelayer_afterr   r   s        r#   merge_recursivelyBMixedModel._unload_and_optionally_merge.<locals>.merge_recursively   s    DE%..E"(( %.. .1crDH-E)!!Z}!U*5*@*@' .F LLJmLLr%   c              3  ,   >#    U  H	  oT;   v   M     g 7fr   rI   rJ   rK   keys     r#   rM   :MixedModel._unload_and_optionally_merge.<locals>.<genexpr>   s     Kqhp^dVYMhprO   z
Unloading zand merging  r   )disabledescrD   r   )rU   r   r+   rX   rY   rZ   r   r   AttributeErrorrR   r`   rS   r(   r   modules_to_saverp   r   rQ   )r   r   progressbarr   r   r   r   _key_listr   r[   r~   target_namer]   s      `` `       r#   _unload_and_optionally_merge'MixedModel._unload_and_optionally_merge   sc    tzz#8$?6I !WXX
	M '+jj&>&>&@r&@FCKqhpKqHqC&@r~B?'I/EC.=djj#.N+ v|,,%f-$$V&:O:O:QSYZF$899#33F4I4IJ
:|44"((Jm(\!+!:!:!<JZ8% F( zz/ s " s   %F?F,F
F F c                H    [        SU R                  R                   S35      e)Nz(Weighted adapters are not supported for rv   )NotImplementedErrorr"   r,   r   r@   rA   s      r#   add_weighted_adapterMixedModel.add_weighted_adapter  s#    !$LT^^MdMdLeel"mnnr%   c           	       ^ [        U[        5      (       a  U/nOUn[        U5      [        U R                  R	                  5       5      -
  nU(       a;  [        S[        U5       S[        U R                  R	                  5       5       35      eU H  nU R                  U	 U R                  R                  5        V^Vs/ s H(  u  mn[        U4S j[         5       5      (       a  M&  TPM*     nnnSnU HX  m[        U R                  T5      u  pXn[        U[        5      (       d  M3  UR                  U5        Ub  MI  UR                  SS nMZ     M     W=(       d    / U l        [!        U R                  XS9  gs  snnf )zq
Deletes an existing adapter.

Args:
    adapter_name (Union[str, list[str]]): Name of the adapter(s) to delete.
zAdapter(s) z  not found, available adapters: c              3  ,   >#    U  H	  oT;   v   M     g 7fr   rI   r   s     r#   rM   ,MixedModel.delete_adapter.<locals>.<genexpr>  s     OultbhZ]P]ltrO   N)new_active_adapters)r(   strr   rl   keysr+   sortedr   rX   rY   rZ   r   r   delete_adapterrk   rp   r   )	r   r!   r   
mismatchedr   r   r   new_adapterr~   s	       `    r#   r   MixedModel.delete_adapter  sE    lC(()NM(M'#d.>.>.C.C.E*FF
fZ011QRXY]YiYiYnYnYpRqQrs  *L  .*.***B*B*Dv*DQCOultOuLu*DHvK.tzz3?1fn55)),7"*&,&<&<Q&?   * */R!$**l\ ws   >%E9'E9c                :    U R                   R                  " U0 UD6$ r   )r   generater   s      r#   r   MixedModel.generate'  s    zz""D3F33r%   )rp   )r   	nn.Moduler    r)   r!   r   returnNone)r    r)   r   r   )r    r)   r@   r   rA   r   r   r   )r   r   )r   r   r   r   )F)r!   Union[str, list[str]]r   boolr   r   )TFFN)r   r   r   r   r   zOptional[list[str]])r@   r   rA   r   r   r   )r!   r   r   r   )r@   r   rA   r   )r,   
__module____qualname____firstlineno____doc__r   r/   r5   r`   rr   staticmethodr}   r   r   r   r   r   r   __static_attributes____classcell__)r"   s   @r#   r   r   7   s    62tt t 	t
 
t*/>Q,  6+   ! -1. . 	.
 +.`o]B4 4r%   r   )9
__future__r   r   typingr   r   r   torchr   r   peft.tunersr	   r
   r   r   r   r   peft.tuners.tuners_utilsr   r   r   
peft.utilsr   r   r   r   r   LORALOHALOKRADALORAOFTSHIRAr-   r7   rK   r;   r9   r=   r?   rZ   r6   r8   r:   r3   r<   r>   r)   r   	LoraLayer	LoHaLayer	LoKrLayerAdaLoraLayerOFTLayer
ShiraLayerrn   r   rI   r%   r#   <module>r      sB   #  ' '   = = Y Y  #--xGWGWYaYeYegoguguv NNNNNNLL	 OOT__doow7L7Lcmm]b]n]nn 	JJJJJJMMLL	
q4 q4r%   