
    oi@:                     X   S SK r S SKJrJr  S SKrS SKrS SKrS SKJr  S SKJ	r	  S SK
Jr  S SKJr  \R                  R                  \R                   5      \R                  R                  S5      :  a  S SKJr  OS SKJr   " S	 S
\	5      r " S S\R*                  \5      r " S S5      rg)    N)AnyOptional)nn)	LoraLayer)check_adapters_to_merge)	transposez4.33.0)deepspeed_configc                   j   ^  \ rS rSrSrSrS\R                  SS4U 4S jjr SS\	4S	 jjr
S
 rSrU =r$ )AdaLoraLayer"   )lora_Alora_Blora_Elora_embedding_Alora_embedding_B)r
lora_alphascalinglora_dropoutranknum
base_layerreturnNc                    > [         TU ]  U5        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        g N)super__init__r   ParameterDictr   r   r   r   )selfr   	__class__s     S/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/adalora/layer.pyr   AdaLoraLayer.__init__)   sX    $&&r*&&r*&&r*''+    inference_modec                    US:  a  [        SU 35      eX R                  U'   X0R                  U'   US:  a  [        R                  " US9nO[        R
                  " 5       nXR                  U'   [        R                  " [        R                  " X R                  5      5      U R                  U'   [        R                  " [        R                  " US5      5      U R                  U'   [        R                  " [        R                  " U R                  U5      5      U R                  U'   [        R                  " [        R                  " S5      SS9U R                  U'   U R                  U   R                   R#                  [%        U5      5        SU R                  U   l        US:  a  UO
[%        U5      U R(                  U'   U(       a  U R+                  U5        U R-                  U5        U R/                  U R0                  US9  g )	Nr   z?`r` should be a positive integer or 0, but the value passed is         )p   F)requires_grad)r#   )
ValueErrorr   r   r   DropoutIdentityr   	Parametertorchrandnin_featuresr   r   out_featuresr   r   datafill_floatr(   r   reset_lora_parameters%_move_adapter_to_device_of_base_layerset_adapteractive_adapters)	r   adapter_namer   r   r   init_lora_weightsr#   kwargslora_dropout_layers	            r    update_layerAdaLoraLayer.update_layer0   s    q5^_`^abcc |(2%#!#l!;!#*<,' %'LLQ@P@P1Q$RL!$&LLQ1B$CL!$&LLT=N=NPQ1R$SL!%'\\%++a.PU%V\"\"''--eAh738\"03=>ZuQx\"&&|422<@--nMr"   c                 H   XR                   R                  5       ;   a  [        R                  R	                  U R
                  U   5        [        R                  R                  U R                   U   SSS9  [        R                  R                  U R                  U   SSS9  g g )Nr%   g{Gz?)meanstd)r   keysr   initzeros_r   normal_r   )r   r8   s     r    r4   "AdaLoraLayer.reset_lora_parametersQ   sq    ;;++--GGNN4;;|45GGOODKK5CTOJGGOODKK5CTOJ .r"   )r   r   r   r   F)__name__
__module____qualname____firstlineno__adapter_layer_namesother_param_namesr   Moduler   boolr<   r4   __static_attributes____classcell__r   s   @r    r   r   "   sL     aQ,299 , , diN\`NBK Kr"   r   c                     ^  \ rS rSr     SS\R
                  S\S\S\S\S\	S\	S	S
4U 4S jjjr
SS\	S\\\      S	S
4S jjrSS jrS	\R                   4S jrS\R                   S\S\S	\R                   4S jrS	\4U 4S jjrSrU =r$ )	SVDLinearX   r   r8   r   r   r   fan_in_fan_outr9   r   Nc                    > [         T	U ]  5         [        R                  X5        SU R                  5       R                  l        X`l        X l        U R                  X#XEU5        g )NF)	r   r   r   get_base_layerweightr(   rU   _active_adapterr<   )
r   r   r8   r   r   r   rU   r9   r:   r   s
            r    r   SVDLinear.__init__Z   sT     	d/5:$$2,+,:EVWr"   
safe_mergeadapter_namesc                 >   [        X5      nU(       d  gU GH  nU R                  5       nX0R                  R                  5       ;   d  M3  U(       a  UR                  R
                  R                  5       nXPR                  U5      -  n[        R                  " U5      R                  5       (       d  [        SU S35      eXTR                  l        O.UR                  =R
                  U R                  U5      -  sl        U R                  R                  U5        GM     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`List[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
Nz1NaNs detected in the merged weights. The adapter z seems to be broken)r   rW   r   rA   rX   r1   cloneget_delta_weightr-   isfiniteallr)   merged_adaptersappend)r   r[   r\   active_adapterr   orig_weightss         r    mergeSVDLinear.mergen   s     0D+N,,.J!1!1!33 $.#4#4#9#9#?#?#AL $9$9.$IIL >>,7;;==(OP^O__rs  .:%%*%%**d.C.CN.SS*$$++N;# ,r"   c                    U R                   (       d  [        R                  " S5        g[        U R                  5      S:  a  U R                  R                  5       nXR                  R                  5       ;   a<  U R                  5       R                  =R                  U R                  U5      -  sl
        [        U R                  5      S:  a  M  gg)zG
This method unmerges all merged adapter layers from the base weights.
z Already unmerged. Nothing to do.Nr   )mergedwarningswarnlenrb   popr   rA   rW   rX   r1   r_   )r   rd   s     r    unmergeSVDLinear.unmerge   s     {{MM<=$&&'!+!11557N!1!1!33##%,,11T5J5J>5ZZ1 $&&'!+r"   c                     [        U R                  U   U R                  U   U R                  U   -  -  U R                  5      U R
                  U   -  U R                  U   S-   -  $ Ngh㈵>)r   r   r   r   rU   r   r   )r   adapters     r    r_   SVDLinear.get_delta_weight   se    dkk'*dkk'.BT[[QXEY.YZ\`\o\opll7#$||G$t+-	
r"   xargsr:   c                    U R                   (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ U R                  " U/UQ70 UD6nU R                   H  nXPR
                  R                  5       ;  a  M"  U R
                  U   nU R                  U   nU R                  U   nU R                  U   n	U R                  U   n
U R                  U   S-   nU R                  XR                  5      nXI" U5      Xh-  R                  -  UR                  -  U
-  U-  -  nM     U$ rq   )disable_adaptersri   rn   r   r7   r   rA   r   r   r   r   r   _cast_input_dtypedtypeT)r   rt   ru   r:   resultrd   r   r   r   dropoutr   r   s               r    forwardSVDLinear.forward   sG     {{__Q888F$ # [[__Q888F   __Q888F"&"6"6!)9)9);;^4^4^4++N;,,~6,,~6=**1ll;71:(;(;;fhhF'QT[[[ #7 r"   c                 *   > [         TU ]  5       nSU-   $ )Nzadalora.)r   __repr__)r   repr   s     r    r   SVDLinear.__repr__   s    g Cr"   )rY   rU   )r   r'   r%   FT)FN)r   N)rG   rH   rI   rJ   r   rM   strintr3   rN   r   r   listrf   rn   r-   Tensorr_   r   r}   r   rO   rP   rQ   s   @r    rS   rS   X   s     !$"&XIIX X 	X
 X X X  X 
X X(#< #<Xd3i=P #<\` #<J
[
5<< 
 c S U\\ 0 #    r"   rS   c                   f    \ rS rSrSrS rS rS rS rS\	4S jr
S	 rS
 rS rS rSS jrS rSrg)RankAllocator   z
The RankAllocator for AdaLoraModel. Paper: https://openreview.net/pdf?id=lq62uWRJjiY

Args:
    config ([`AdaLoraConfig`]): The configuration of the AdaLora model.
    model: the model that we apply AdaLoRA to.

c                 *   X l         X0l        UR                  U l        UR                  U l        U R                  S:  a  U R                  S:  d   eU R                  S:  a  U R                  S:  d   eU R	                  5         U R                  U5        g )Nr   r'   )peft_configr8   beta1beta2	reset_ipt_set_budget_scheduler)r   modelr   r8   s       r    r   RankAllocator.__init__   sv    &( &&
 &&
zzA~$**q.00zzA~$**q.00""5)r"   c                 $    XR                   l        g r   )r   
total_step)r   r   s     r    set_total_stepRankAllocator.set_total_step   s    &0#r"   c                 .    0 U l         0 U l        0 U l        g r   )iptexp_avg_iptexp_avg_unc)r   s    r    r   RankAllocator.reset_ipt   s    r"   c                    SU l         [        5       U l        UR                  5        Hi  u  p#SU R                   3U;   d  M  U =R                   UR                  S5      -  sl         U R                  R                  UR                  SS5      5        Mk     [        U R                  5      U l        U R                  R                  [        U R                  5      -  U l        g )Nr   lora_A.r   %s)init_bgtsetname_setnamed_parametersr8   sizeaddreplacesortedr   target_rrl   
target_bgt)r   r   nr&   s       r    r   #RankAllocator._set_budget_scheduler   s    **,DA**+,1*!!!))Hd";< - t}}-**33c$--6HHr"   stepc                    U R                   R                  nU R                   R                  nU R                   R                  nX::  a  U R                  nSnXV4$ XU-
  :  a  U R
                  nSnXV4$ SX-
  XC-
  U-
  -  -
  n[        U R                  U R
                  -
  US-  -  U R
                  -   5      nXR                   R                  -  S:X  a  SOSnXV4$ )NFTr'      r   )r   tinittfinalr   r   r   r   deltaT)r   r   r   r   r   budgetmask_ind	mul_coeffs           r    budget_scheduleRankAllocator.budget_schedule   s      &&!!((%%00
=]]FH  ''__FH  T\j.AE.IJJI$--$//9ilKdoo]^F#&6&6&=&==BtHr"   c                 >   UR                  5        GH  u  p#SU;   d  M  U R                  U;   d  M   X R                  ;  ai  [        R                  " U5      U R                  U'   [        R                  " U5      U R
                  U'   [        R                  " U5      U R                  U'   [        R                  " 5          [        5       bM  SS K	nUR                  R                  U5      nX5-  R                  5       R                  5       U R                  U'   O7X3R                  -  R                  5       R                  5       U R                  U'   U R                  U R
                  U   -  SU R                  -
  U R                  U   -  -   U R
                  U'   U R                   U R                  U   -  SU R                   -
  U R                  U   U R
                  U   -
  R                  5       -  -   U R                  U'   S S S 5        GM     g ! , (       d  f       GM  = f)Nlora_r   r'   )r   r8   r   r-   
zeros_liker   r   no_gradr	   	deepspeedutilssafe_get_full_gradabsdetachgradr   r   )r   r   r   r&   r   r   s         r    
update_iptRankAllocator.update_ipt   s   **,DA!| 1 1Q 6HH$"'"2"21"5DHHQK*/*:*:1*=D$$Q'*/*:*:1*=D$$Q']]_')5((AA!D'(xnn&6&=&=&?'(66z&6&6&8&?&?&A*.**t7G7G7J*JaRVR\R\n`d`h`hij`kMk*kD$$Q' 

T%5%5a%88A

NtxxXY{]a]m]mno]pOpNuNuNw;ww $$Q' %_ - %_s   >E H
H	c                 @    U R                   U   U R                  U   -  $ r   )r   r   )r   r   s     r    _element_scoreRankAllocator._element_score  s#    "T%5%5a%888r"   c                 l    UR                  SSS9nUR                  S5      UR                  S5      -   nU$ )Nr'   Fdimkeepdim)sumview)r   ipt_Eipt_ABsum_ipts       r    _combine_iptRankAllocator._combine_ipt  s3    51**R.6;;r?2r"   c                 4   0 n0 n0 nUR                  5        GH&  u  pgSU R                   3U;   aW  U R                  U5      n[        R                  " USSS9n	UR                  SS5      n
X;  a  U	/XJ'   OXJ   R                  U	5        SU R                   3U;   ag  U R                  U5      n[        R                  " USS	S9R                  S
S5      n	UR                  SS5      n
X;  a  U	/XJ'   OXJ   R                  U	5        SU R                   3U;   d  M  U R                  U5      nUR                  SS5      n
XU
'   GM)     / nU Hh  n
X:   n[        R                  " XJ   SS9nU R                  X5      nU
S-  nUR                  S
S5      X_'   UR                  UR                  S
5      5        Mj     [        R                  " [        R                  " U5      U R                  U-
  S9S   R                  5       n0 n[        R                  " 5          UR                  5        HY  u  pgSU R                   3U;   d  M  UR                  XV   U:*  S5        XV   U:*  ) R                  S
5      R                  5       UU'   M[     S S S 5        U$ ! , (       d  f       U$ = f)Nr   r'   Tr   r   r   zlora_B.r   Fr   r   lora_E.r   )r   )kr%   )r   r8   r   r-   r?   r   rc   r   catr   kthvaluer   itemr   masked_fill_tolist)r   r   r   	value_ipt
vector_ipttriplet_iptr   r&   	entry_iptcomb_iptname_m	all_scorer   r   r   name_Emask_thresholdrank_patterns                     r    mask_to_budgetRankAllocator.mask_to_budget  s   	
**,DA**+,1 //2	 ::iQE8T2+*2J&&--h7**+,1 //2	 ::iQFKKBPQR8T2+*2J&&--h7**+,1 //2	8T2$-&!) -, 	 F%EYYz1q9F''6Gh&F"),,r1"5KW\\"-. ! IIi mmf$
  TV 	
 ]]_..0T../0A5NN;>^#CSI)4>)I'J&P&PQS&T&[&[&]LO 1 
  _
 s   'J;AJ
Jc                     X R                   R                  U R                   R                  -
  :  a  U R                  U5        U R	                  U5      u  pEU(       d  U(       a  U R                  X5      nXF4$ S nXF4$ r   )r   r   r   r   r   r   )r   r   global_step
force_maskr   r   r   s          r    update_and_allocate!RankAllocator.update_and_allocateQ  st    ))44t7G7G7N7NNNOOE"//<z..u=L ##  L##r"   c                 8   SnU R                   [        [        UR                  5       5      5      ;  a  Sn[        R
                  " 5          UR                  5        H  u  pESU R                    3U;   d  M  U(       d  UOUR                  SU R                    3S5      n[        R                  " X&   5      R                  S5      R                  UR                  5      nUR                  UR                  5       ) S5        M     S S S 5        g ! , (       d  f       g = f)NFTr   . r   r%   )r8   nextiterrA   r-   r   r   r   r   	unsqueezetodevicer   rN   )r   r   r   is_adapter_name_truncatedr   r&   keymasks           r    mask_using_rank_pattern%RankAllocator.mask_using_rank_pattern]  s    $)!Dl.?.?.A)B$CC(,%]]_..0T../0A5#<!!))aPTPaPaObLcegBhC <<(9:DDRHKKAHHUDNNDIIK<5	 1 __s   
'D5BD
D)
r8   r   r   r   r   r   r   r   r   r   NrF   )rG   rH   rI   rJ   __doc__r   r   r   r   r   r   r   r   r   r   r   r   rO    r"   r    r   r      sG    	*1
	I C  &.9
2h
$6r"   r   )rj   typingr   r   	packagingr-   transformersr   peft.tuners.lorar   peft.tuners.tuners_utilsr   
peft.utilsr   versionparse__version__transformers.integrationsr	   transformers.deepspeedr   rM   rS   r   r   r"   r    <module>r     s           & <   <334	8I8I8O8OPX8YY:73K9 3Klh 		< h Ve6 e6r"   