
    oi)<                         S SK r S SKJr  S SKrS SKJr  S SKJs  Jr  S SK	J
r
  S SKJrJr  S SKJr  SSKJr   " S S	\R$                  R&                  5      r " S
 S\5      r " S S\R,                  \5      rg)    N)Optional)Conv1D)BaseTunerLayercheck_adapters_to_merge)	transpose   )
BufferDictc                   4    \ rS rSr\S 5       r\S 5       rSrg)UniqueBaseGrad   c                 T    US S 2S S 2S 4   U-  US   -  nU R                  XU5        U$ )NN)save_for_backward)ctx
randlora_Arandlora_lambdarandlora_gammaouts        T/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/randlora/layer.pyforwardUniqueBaseGrad.forward   s6    aDj)J69NNj>J
    c                 (   U R                   u  p#nUR                  UR                  5      UR                  UR                  5      UR                  UR                  5      pCn[        R                  " SXU5      n[        R                  " SXU5      nS XV4$ )Nzkbj,kvj,bj->kbzkbj,kvj,kb->bj)saved_tensorstodtypetorcheinsum)r   grad_outputr   r   r   grad_randlora_lambdagrad_randlora_gammas          r   backwardUniqueBaseGrad.backward%   s    696G6G3
^MM+++,{001k//0 &4

  %||,<kWef#ll+;[Vef)>>r    N)__name__
__module____qualname____firstlineno__staticmethodr   r"   __static_attributes__r$   r   r   r   r      s(     
 	? 	?r   r   c                   x    \ rS rSrSrSrS\R                  4S jr\	S\
4S j5       r SS\S	\S
\
4S jjrS rSrg)RandLoraLayer2   )r   r   )r   
randlora_B
base_layerc                    Xl         0 U l        0 U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        S U l	        S U l
        SU l        / U l        SU l        U R                  5       n[        U[        R                   5      (       a  UR"                  UR$                  pCO^[        U[&        5      (       aI  [)        UR*                  S5      (       a  UR*                  R,                  OUR*                  R.                  u  p4WU l        WU l        X l        g )NFTds_shape)r/   rscalingnn
ModuleDictrandlora_dropoutParameterDictr   r   r   r.   _disable_adaptersmerged_adapterscast_input_dtype_enabledget_base_layer
isinstanceLinearin_featuresout_featuresr   hasattrweightr1   shapekwargs)selfr/   rC   r>   r?   s        r   __init__RandLoraLayer.__init__7   s   $ "b 1  "//3 ..r2 1504 "'! )-%((*
j")),,(2(>(>
@W@W
F++.5j6G6G.T.T
!!**ZdZkZkZqZq &K '(r   returnc                 ,    [        U R                  5      $ r   )boolr9   )rD   s    r   mergedRandLoraLayer.mergedY   s    D(())r   r   r.   inference_modec	           	         US::  a  [        SU 35      eX@R                  U'   US:  a  [        R                  " US9n
O[        R                  " 5       n
U R
                  R                  [        R                  " X05      5        [        U R                  U R                  5      U-  nUR                  5       (       a  [        U5      O[        U5      S-   U l        [        R                  " [        R                   " X@R                  5      SS9U R"                  U'   [        R                  " [        R$                  " U R                  [        U R                  U R                  5      5      ['        U R                  U R                  5      -  SS9U R(                  U'   XT-  U R*                  U'   X l        X0l        X;  Ga  [1        U R,                  5      S:  a  [        S5      e[3        U R,                  R5                  5       5      S   n[3        U R.                  R5                  5       5      S   nS	n['        U R                  U R                  5      [        U R                  U R                  5      nnUR6                  S   U:  a)  [        UR9                  S
UR6                  S   U5      5      eUR6                  S   U:  a)  [        UR9                  SUR6                  S   U5      5      eSnUR6                  S   U R                  U   :  a6  [        UR9                  SUR6                  S   U R                  U   5      5      eUR6                  S   U R                  U   :  a6  [        UR9                  S
UR6                  S   U R                  U   5      5      eXR,                  U'   XR.                  U'   U(       a  U R;                  U5        U R=                  U5        U R?                  U R@                  US9  g )Nr   z?`r` should be a positive integer value but the value passed is         )p   T)requires_gradzfThe `randlora_A` and `randlora_B` buffers are empty. This should not happen. Please report this issue.z{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter was added after the first one with incompatible shapes.r.   r   z{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter with a lower rank was added after the first one; loading the adapters in reverse order may solve this.)rL   )!
ValueErrorr2   r4   DropoutIdentityr6   updater5   minr>   r?   
is_integerint	num_bases	Parameterr   randnr   onesmaxr   r3   r   r.   lenlistvaluesrB   formatreset_randlora_parameters%_move_adapter_to_device_of_base_layerset_adapteractive_adapters)rD   adapter_namer   r.   r2   randlora_alphar6   init_weightsrL   rC   randlora_dropout_layerrZ   randlora_A_paramrandlora_B_param
error_tmplmax_dimmin_dims                    r   update_layerRandLoraLayer.update_layer]   sj    6^_`^abcc |c!%'ZZ2B%C"%'[[]"$$R]]L3Y%Z[ (($*;*;<q@	+4+?+?+A+AYs9~XYGY-/\\%++a:Xhl-m\*,.LLJJt~~s4+<+<d>N>N'OP$##T%5%567-
L) &4%7\" %$)4??#a' |   $DOO$:$:$<=a@#DOO$:$:$<=a@R   #4#3#3T5F5FGTM]M]_c_p_pIqWG%%a(72 !2!2<AQAWAWXYAZ\c!dee%%b)G3 !2!2<AQAWAWXYAZ\c!dee3 
  %%a(466,+?? !2!2<AQAWAWXYAZ\`\b\bco\p!qrr%%b)DFF<,@@ !2!2<AQAWAWXZA[]a]c]cdp]q!rss,<OOL),<OOL)**<822<@--nMr   c           	         XR                   R                  5       ;   a  [        R                  " 5          [        R
                  R                  U R                   U   5        [        R
                  R                  U R                  U   S[        U R                  U   R                  5      -  5        S S S 5        g g ! , (       d  f       g = f)NrP   )r   keysr   no_gradr4   initzeros_	constant_r   r^   rB   )rD   rg   s     r   rc   'RandLoraLayer.reset_randlora_parameters   s    //4466t33LAB!!$"5"5l"CQTM`M`amMnMtMtIuEuv ! 7 s   A=B::
C)r8   r/   r:   r>   rC   r9   rZ   r?   r2   r   r.   r6   r   r   r3   N)F)r%   r&   r'   r(   adapter_layer_namesother_param_namesr4   ModulerE   propertyrI   rJ   r	   rp   rc   r*   r$   r   r   r,   r,   2   sq    ?4 299  D * * *  %NN NN 	NN NN`wr   r,   c                   H  ^  \ rS rSr      SS\S\S\S\S\S\S\S	\S
\SS4U 4S jjjr	SS\S\
\\      SS4S jjrSS jrSS\\R                   \R                   4   4S jjrS\R                   4S jrS\R                   S\R                   4S jrS\4U 4S jjrSrU =r$ )r=      r   r.   rg   r2   rh   r6   fan_in_fan_outis_target_conv_1d_layerri   rG   Nc           	         > [         [        R                  U ]  5         [        R                  " X40 UD6  Xl        X@l        U R                  XBX5XgU
5        Xl        g r   )	superr4   r=   rE   r,   r   _active_adapterrp   r   )rD   r/   r   r.   rg   r2   rh   r6   r   r   ri   rC   	__class__s               r   rE   Linear.__init__   sP     	bii')t:6:,+,J>eqr'>$r   
safe_mergeadapter_namesc                    [        X5      nU(       d  gU GH:  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  nU(       a  UR                  R                  R                  5       nX`R                  U5      -  n[        R                  " U5      R                  5       (       d  [        SU S35      eUR                  U5      UR                  l        O?U R                  U5      nUR                  =R                  UR                  U5      -  sl        U R                  R                  U5        GM=     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
Nz1NaNs detected in the merged weights. The adapter z seems to be broken)r   r   rs   r;   rA   r   datacloneget_delta_weightr   isfiniteallrS   r   r9   append)rD   r   r   active_adapterr/   
orig_dtypeorig_weightsdelta_weights           r   mergeLinear.merge   s    0D+N!5!5!:!:!<<!002
'..44
 $.#4#4#9#9#?#?#AL $9$9.$IIL >>,7;;==(OP^O__rs  .:__Z-HJ%%*#'#8#8#HL%%**looj.II*$$++N;- ,r   c                    U R                   (       d  [        R                  " S5        g[        U R                  5      S:  a  U R                  5       nUR                  R                  nU R                  R                  5       nX0R                  R                  5       ;   a?  U R                  U5      nUR                  =R                  UR                  U5      -  sl        [        U R                  5      S:  a  M  gg)zG
This method unmerges all merged adapter layers from the base weights.
z Already unmerged. Nothing to do.Nr   )rJ   warningswarnr_   r9   r;   rA   r   popr   rs   r   r   r   )rD   r/   r   r   r   s        r   unmergeLinear.unmerge   s     {{MM<=$&&'!+,,.J#**00J!11557N!5!5!:!:!<<#44^D!!&&,//**EE& $&&'!+r   c                    U R                   U   nU R                  U   nUc  UR                  nUR                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                  U   R                  U5      nU R                  U   R                  U5      nU(       a@  UR                  5       nUR                  5       nUR                  5       nUR                  5       n[        U R                  U R                  5      [        U R                  U R                  5      pUSS2SU R                   2SU	24   R                  U5      nUSU
2SU R                   2SS24   R                  U5      nUR#                  SS9n[$        R'                  XU5      R#                  SS9nXR                  :X  a  X4$ UR(                  UR(                  4$ )a  
Performs scaling on the smallest random base (randlora_A) and returns randlora_A and randlora_B in the correct
order to fit the target layers' dimensions

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
NcpurP   )	start_dim)end_dim)r   r.   devicer   typer   float16bfloat16r   r   r   floatrW   r?   r>   r^   rZ   flattenr   applyT)rD   adapterr   r   r.   r   cast_to_fp32r   r   ro   rn   sliced_Asliced_Bupdate_Bupdate_As                  r   get_scaled_basesLinear.get_scaled_bases  s    __W-
__W-
>&&F  
 {{e+c%--1G1b5TYTbTbKb..w7::6B,,W588@#))+J#))+J-335O+113N t00$2B2BCSIZIZ\`\l\lEm
 a!14>>!18G8;<??Ghwh(8$..(8!;<??G ##a#0!''>RZZcdZe &&&%%zz8::%%r   c                     U R                  U5      u  p#UR                  UR                  -  R                  n[        X@R                  5      nU R                  U   nXV-  $ )z
Compute the delta weight for the given adapter.

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
)r   r   r   r   r3   )rD   r   r   r   rV   output_tensorr3   s          r   r   Linear.get_delta_weight6  sV     "227;**xzz),,!&*=*=>,,w'&&r   xc           	         UR                   nU R                  (       a8  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nOU R                  (       a  U R                  " U/UQ70 UD6nOU R                  " U/UQ70 UD6nU R
                   H  nX`R                  R                  5       ;  a  M"  U R                  U   nU R                  XaR                  S9u  pUR                  U	R                   5      nU R                  U   n
U[        R                  " [        R                  " U" U5      U5      U	5      U
-  -   nM     UR                  U5      nU$ )N)r   )r   disable_adaptersrJ   r   r/   rf   r   rs   r6   r   r   r   r3   Flinear)rD   r   argsrC   previous_dtyperesultr   dropoutr   r   r3   s              r   r   Linear.forwardG  s&     {{__Q888F[[__Q888F__Q888F"&"6"6!)=)=)B)B)DD//?%)%:%:>RZRZ%:%["DD(,,~6!((188GAJ+I8"TW^"^^ #7 >*r   c                 *   > [         TU ]  5       nSU-   $ )Nz	randlora.)r   __repr__)rD   repr   s     r   r   Linear.__repr__\  s    g S  r   )r   r   r   )r   r   rN   FFT)FN)rG   Nr   )r%   r&   r'   r(   r	   strrY   r   rI   rE   r   r`   r   r   tupler   Tensorr   r   r   r   r*   __classcell__)r   s   @r   r=   r=      s    "%$(-!? ? 	?
 ? ? ?  ? ? "&? ? 
? ?,(< (<Xd3i=P (<\` (<TF .&ellELL>X8Y .&`'5<< '" 5<< *!# ! !r   r=   )r   typingr   r   torch.nnr4   torch.nn.functional
functionalr   transformers.pytorch_utilsr   peft.tuners.tuners_utilsr   r   peft.utils.otherr   _buffer_dictr	   autogradFunctionr   r,   r=   r$   r   r   <module>r      s`          - L & %?U^^,, ?*wN wDj!RYY j!r   