
    oiJ                         S SK r S SKrS SKJrJrJr  S SKrS SKJr  S SK	Js  J
r  S SKJrJr   " S S\5      r " S S\R                   \5      r " S S	\R                   \5      rg)
    N)AnyOptionalUnion)BaseTunerLayercheck_adapters_to_mergec                       \ rS rSrSrSrS\R                  SS4S jr SS\	S	\
S
\S\S\SS4S jjrS\	4S jrS\	4S jrS\SS4S jrSSS jjrSrg)HRALayer   )hra_u)hra_rhra_apply_GS
base_layerreturnNc                    Xl         0 U l        0 U l        [        R                  " 0 5      U l        SU l        / U l        SU l        X l	        U R                  5       n[        U[        R                  5      (       a$  UR                  UR                  sU l        U l        g [        U[        R                  5      (       a$  UR                   UR"                  sU l        U l        g [%        S['        U5       35      e)NFTzUnsupported layer type )r   r   r   nnParameterDictr   _disable_adaptersmerged_adapterscast_input_dtype_enabledkwargsget_base_layer
isinstanceLinearin_featuresout_featuresConv2din_channelsout_channels
ValueErrortype)selfr   r   s      O/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/hra/layer.py__init__HRALayer.__init__    s    $
%%b)
!&!(,%((*
j")),,2<2H2H*JaJa/Dd/
BII..2<2H2H*JaJa/Dd/6tJ7G6HIJJ    adapter_namerapply_GSinit_weightsinference_modec                 4   US::  a  [        SU 35      eX R                  U'   X0R                  U'   U R                  5       n[	        U[
        R                  5      (       aB  [
        R                  " [        R                  " U R                  U5      SS9U R                  U'   O[	        U[
        R                  5      (       ab  [
        R                  " [        R                  " U R                  UR                  S   -  UR                  S   -  U5      SS9U R                  U'   O![        S[        U5      R                    35      eU(       a  U R#                  U5        OU R%                  U5        U R'                  U5        U R)                  U R*                  US9  g)a  Internal function to create hra adapter

Args:
    adapter_name (`str`): Name for the adapter to add.
    r (`int`): Rank for the added adapter.
    init_weights (`bool`): Whether to initialize weights.
    apply_GS (`bool`): Whether to apply Gram-Schmidt orthogonalization or not.
r   z?`r` should be a positive integer value but the value passed is T)requires_gradz/HRA is not implemented for base layers of type )r*   N)r   r   r   r   r   r   r   	Parametertorchemptyr   r   r   kernel_size	TypeErrorr    __name__reset_hra_parametersreset_hra_parameters_random%_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r!   r&   r'   r(   r)   r*   r   r   s           r"   update_layerHRALayer.update_layer4   sT   " 6^_`^abcc#$

< *2,' ((*
j")),,')||EKK@P@PRS4Tdh'iDJJ|$
BII..')||D,,z/E/Ea/HH:KaKabcKddfgh"(DJJ|$
 MdS]N^NgNgMhijj %%l3,,\: 	22<@--nMr%   c                     U R                   U   S-  S:w  aV  [        R                  " S5        [        R                  R                  U R                  U   [        R                  " S5      S9  g U R                  U   R                  n[        R                  " US   US   S-  5      n[        R                  R                  U[        R                  " S5      S9  [        R                  " [        R                  " USSS95      U R                  U'   g )N   r   z@The symmetric initialization can NOT be performed when r is odd!   a   dim)r   warningswarnr   initkaiming_uniform_r   mathsqrtshaper.   zerosr-   repeat_interleave)r!   r&   rH   half_us       r"   r3   HRALayer.reset_hra_parametersa   s    ::l#a'1,MM\]GG$$TZZ%=1$NJJ|,22E[[q58q=9FGG$$Vtyy|$<')||E4K4KFTU[\4]'^DJJ|$r%   c                     [         R                  R                  U R                  U   [        R
                  " S5      S9  g )Nr<   r=   )r   rD   rE   r   rF   rG   )r!   r&   s     r"   r4   $HRALayer.reset_hra_parameters_randomk   s*    
  L!9TYYq\ Jr%   scalec                     US:X  a  g U R                    H8  nX R                  R                  5       ;  a  M"  [        R                  " S5        M:     g )Nr?   zFScaling operation for HRA not supported! Automatically set scale to 1.r7   r   keysrB   rC   r!   rO   active_adapters      r"   scale_layerHRALayer.scale_layern   s=    A:"22NZZ__%66MMbc	 3r%   c                     U R                    H8  nX R                  R                  5       ;  a  M"  [        R                  " S5        M:     g )Nz>Unscaling operation for HRA not supported! Keeping scale at 1.rQ   rS   s      r"   unscale_layerHRALayer.unscale_layerx   s3    "22NZZ__%66MMZ[	 3r%   )
r   r   r   r   r   r   r   r   r   r   FNr   N)r2   
__module____qualname____firstlineno__adapter_layer_namesother_param_namesr   Moduler#   strintboolr8   r3   r4   floatrU   rX   __static_attributes__ r%   r"   r	   r	      s    $1K299 K4 K4  %+N+N +N 	+N
 +N +N 
+NZ_ _K Kd d4 d\ \r%   r	   c                     ^  \ rS rSrSr   SS\S\S\S\\\4   SS4
U 4S	 jjjr	SS
\S\
\\      SS4S jjrSS jrSS\S\S\R                  4S jjrS\R                  S\S\S\R                  4S jrS\4U 4S jjrSrU =r$ )	HRALinear   z#
HRA implemented in a dense layer.
r&   r'   r(   r)   r   Nc                    > [         TU ]  5         [        R                  " X40 UD6  X l        U R                  " X#XE40 UD6  g r[   superr#   r	   _active_adapterr8   r!   r   r&   r'   r(   r)   r   	__class__s          r"   r#   HRALinear.__init__   s>     	$5f5+,8LVLr%   
safe_mergeadapter_namesc                 t   [        X5      nU(       d  gU GH  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  nU(       a  UR                  R                  R                  5       nU R                  U5      n[        R                  " UR                  UR
                  5      U5      n[        R                  " U5      R                  5       (       d  [        SU S35      eUR                  U5      UR                  l        OuU R                  U5      n[        R                  " UR                  R                  R                  UR
                  5      U5      nUR                  U5      UR                  l        U R                  R!                  U5        GM     g)  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`List[str]`, *optional*):
        The list of adapter names that should be merged. If `None`, all active adapters will be merged.
        Defaults to `None`.
N1NaNs detected in the merged weights. The adapter  seems to be broken)r   r   rR   r   weightdtypedatacloneget_delta_weightr.   mmtoisfiniteallr   r   append)	r!   rs   rt   rT   r   
orig_dtypeorig_weightdelta_weight
new_weights	            r"   mergeHRALinear.merge   sR    0D+N!22!002
'..44
 #-"3"3"8"8">">"@K#'#8#8#HL"'((;>>,:L:L+M|"\K >>+6::<<(OP^O__rs  .9^^J-GJ%%*#'#8#8#HL!&**;*;*@*@*C*CLDVDV*WYe!fJ-7]]:-FJ%%*$$++N;+ ,r%   c                    U R                   (       d  [        R                  " S5        g[        U R                  5      S:  a  U R                  R                  5       nU R                  5       nUR                  R                  nXR                  R                  5       ;   a  UR                  R                  R                  5       nU R                  USS9n[        R                  " UR!                  UR                  5      U5      nUR!                  U5      UR                  l        [        U R                  5      S:  a  M  ggzG
This method unmerges all merged adapter layers from the base weights.
z Already unmerged. Nothing to do.Nr   T)reverse)mergedrB   rC   lenr   popr   ry   rz   r   rR   r{   r|   r}   r.   r~   r   )r!   rT   r   r   r   r   r   s          r"   unmergeHRALinear.unmerge   s     {{MM<=$&&'!+!11557N,,.J#**00J!22(//44::<#44^T4R"XXknn\5G5G&H,W
)3z)B
!!& $&&'!+r%   r   c                    U R                   U   nU R                  U   nU R                  U   nUR                  nU(       Ga  US S 2S4   US S 2S4   R	                  5       -  R                  SS5      /n[        SU5       H|  nUS S 2U4   R                  SS5      n	[        U5       H   n
XU
   R                  5       U	-  Xz   -  -
  n	M"     UR                  XR	                  5       -  R                  SS5      5        M~     [        R                  " USS9n[        R                  " US   UR                  UR                  S9SU-  UR                  5       -  -
  nU$ XUR	                  SS9-  n[        R                  " US   UR                  UR                  S9nU(       a  [        US-
  SS5      nO[        U5      nU H8  nUS S 2U4   R                  SS5      n	USU-  U	-  U	R                  5       -  -
  nM:     U$ Nr   r?   r@   )devicerz   r;   r   r   r   rH   normviewrangetr   r.   cateyer   rz   r!   r&   r   rankr(   opt_urH   ry   iuijindicess               r"   r}   HRALinear.get_delta_weight      zz,'$$\2

<(QT{U1a4[%5%5%77==b!DEF1d^1a4[%%b!,qAay{{}r1VY>>B "rGGI~33B:;	 $
 YYv1-FYYuQxEKKPSTW]S]`f`h`h`jSjjF  JJ1J--EYYuQxEKKPFq"b1+1a4[%%b!,!f*r/BDDF"::  r%   xargsr   c                 f   UR                   nU R                  (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nGOU R                  (       a  U R                  " U/UQ70 UD6nGO[
        R                  " U R                  UR                  S9nU R                   Hc  nXpR                  R                  5       ;  a  M"  U R                  U5      n[
        R                  " UR                  UR                   5      U5      nMe     U R                  5       R                   R"                  n	U R%                  XR                   5      n	[
        R                  " X5      nU R%                  U R                  R&                  UR                   5      n
U R(                  (       a  U R%                  XR                   5      nO=UR                  U R                  5       R                   R"                  R                   5      n[*        R,                  " XU
S9nUR                  U5      nU$ )Nr   )inputry   bias)rz   disable_adaptersr   r   r   r.   r   r   r   r7   r   rR   r}   r~   r   r   ry   r{   _cast_input_dtyper   r   Flinearr!   r   r   r   previous_dtyperesultr   rT   r   r   r   s              r"   forwardHRALinear.forward   s     {{__Q888F[[__Q888F4#3#3AHHEJ"&"6"6!)::#44^D"XXjmmL4F4F&GV
	 #7 --/66;;K00>N>NOK+:J))$//*>*>
@P@PQD,,**1.>.>?DD,,.55::@@AXXAtDF>*r%   c                 *   > [         TU ]  5       nSU-   $ Nzhra.rn   __repr__r!   reprq   s     r"   r   HRALinear.__repr__      g |r%   ro   r   FTFNr\   rZ   r2   r]   r^   r_   __doc__rc   rd   re   r   r#   r   listr   r   r.   Tensorr}   r   r   r   rg   __classcell__rq   s   @r"   rj   rj      s     )-M M 	M
 M D#I&M 
M M'< '<Xd3i=P '<\` '<RC$S 4 ELL < c S U\\ @#  r%   rj   c                     ^  \ rS rSrSr   SS\S\S\S\\\4   4U 4S jjjr	SS	\S
\
\\      SS4S jjrSS jrSS\S\S\R                  4S jjrS\R                  S\S\S\R                  4S jrS\4U 4S jjrSrU =r$ )	HRAConv2di  zHRA implemented in Conv2d layerr&   r'   r(   r)   c                    > [         TU ]  5         [        R                  X5        X l        U R                  " X#XE40 UD6  g r[   rm   rp   s          r"   r#   HRAConv2d.__init__  s9     	$++,8LVLr%   Nrs   rt   r   c                    [        X5      nU(       d  gU GH  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  nU(       GaQ  UR                  R                  R                  5       nUR                  U R                  U R                  UR                  S   -  U R                  R                  S   -  5      nU R                  U5      n[        R                  " UR!                  UR
                  5      U5      nUR                  U R                  U R                  UR                  S   UR                  S   5      n[        R"                  " U5      R%                  5       (       d  ['        SU S35      eUR!                  U5      UR                  l        GOUR                  R                  nUR                  U R                  U R                  U R                  R                  S   -  U R                  R                  S   -  5      nU R                  U5      n[        R                  " UR!                  UR
                  5      U5      nUR                  U R                  U R                  UR                  S   UR                  S   5      nUR!                  U5      UR                  l        U R(                  R+                  U5        GM     g)rv   Nr   rw   rx   )r   r   rR   r   ry   rz   r{   r|   r   r   r   r0   r   r}   r.   r~   r   r   r   r   r   r   )r!   rs   rt   rT   r   r   r   r   s           r"   r   HRAConv2d.merge"  sk    0D+N!22!002
'..44
 #-"3"3"8"8">">"@K"-"2"2))((:+A+A!+DDtGbGbcdGee#K $(#8#8#HL"'((;>>,:L:L+M|"\K"-"2"2))(("..q1"..q1	#K !>>+6::<<(OP^O__rs  .9^^J-GJ%%*","3"3"8"8K"-"2"2))((4??+F+Fq+IIDOOLgLghiLjj#K $(#8#8#HL"'((;>>,:L:L+M|"\K"-"2"2))(("..q1"..q1	#K .9^^J-GJ%%*$$++N;W ,r%   c                    U R                   (       d  [        R                  " S5        g[        U R                  5      S:  Ga  U R                  R                  5       nU R                  5       nUR                  R                  nXR                  R                  5       ;   Ga  UR                  R                  R                  5       nUR                  U R                  U R                  UR                   S   -  UR                   S   -  5      nU R#                  USS9n[$        R&                  " UR)                  UR                  5      U5      nUR                  U R                  U R                  UR                   S   UR                   S   5      nUR)                  U5      UR                  l        [        U R                  5      S:  a  GM  ggr   )r   rB   rC   r   r   r   r   ry   rz   r   rR   r{   r|   r   r   r   r0   r}   r.   r~   r   )r!   rT   r   r   r   r   s         r"   r   HRAConv2d.unmergea  st    {{MM<=$&&'!+!11557N,,.J#**00J!22(//44::<)..%%$$z'='=a'@@:CYCYZ[C\\  $44^T4R#hh{~~l6H6H'I<X)..%%t'7'79O9OPQ9RT^TjTjklTm *5
)C
!!&! $&&'!++r%   r   c                    U R                   U   nU R                  U   nU R                  U   nUR                  nU(       Ga  US S 2S4   US S 2S4   R	                  5       -  R                  SS5      /n[        SU5       H|  nUS S 2U4   R                  SS5      n	[        U5       H   n
XU
   R                  5       U	-  Xz   -  -
  n	M"     UR                  XR	                  5       -  R                  SS5      5        M~     [        R                  " USS9n[        R                  " US   UR                  UR                  S9SU-  UR                  5       -  -
  nU$ XUR	                  SS9-  n[        R                  " US   UR                  UR                  S9nU(       a  [        US-
  SS5      nO[        U5      nU H8  nUS S 2U4   R                  SS5      n	USU-  U	-  U	R                  5       -  -
  nM:     U$ r   r   r   s               r"   r}   HRAConv2d.get_delta_weightz  r   r%   r   r   r   c                    UR                   nU R                  (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nGOU R                  (       a  U R                  " U/UQ70 UD6nGO[
        R                  " U R                  U R                  R                  S   -  U R                  R                  S   -  UR                  S9nU R                   Hc  nXpR                  R                  5       ;  a  M"  U R                  U5      n[
        R                  " UR                  UR                   5      U5      nMe     U R                  R                   R"                  n	U	R%                  U R&                  U R                  U R                  R                  S   -  U R                  R                  S   -  5      n	U R)                  XR                   5      n	U R)                  U R                  R*                  UR                   5      n
[
        R                  " X5      nUR%                  U R&                  U R                  U R                  R                  S   U R                  R                  S   5      nU R,                  (       a  U R)                  XR                   5      nO=UR                  U R/                  5       R                   R"                  R                   5      n[0        R2                  " UUU
U R                  R4                  S   U R                  R6                  S   S9nUR                  U5      nU$ )Nr   r   )r   ry   r   paddingstride)rz   r   r   r   r   r.   r   r   r0   r   r7   r   rR   r}   r~   r   ry   r{   r   r   r   r   r   r   r   conv2dr   r   r   s              r"   r   HRAConv2d.forward  s     {{__Q888F[[__Q888F  4??#>#>q#AADOOD_D_`aDbbxxJ #'"6"6!)::#44^D"XXjmmL4F4F&GV
	 #7 //0055K%**!!  4??#>#>q#AADOOD_D_`aDbbK 00>N>NOK))$//*>*>
@P@PQD+:J#!!  ++A.++A.	J ,,**1.>.>?DD,,.55::@@AXX!//2--a0F >*r%   c                 *   > [         TU ]  5       nSU-   $ r   r   r   s     r"   r   HRAConv2d.__repr__  r   r%   r   r   r   r\   rZ   r   r   s   @r"   r   r     s    ) )-M M 	M
 M D#I&M M=< =<Xd3i=P =<\` =<~D2S 4 ELL <1 1c 1S 1U\\ 1f#  r%   r   )rF   rB   typingr   r   r   r.   torch.nnr   torch.nn.functional
functionalr   peft.tuners.tuners_utilsr   r   r	   rb   rj   r   rh   r%   r"   <module>r      s\      ' '     Lc\~ c\LN		8 Nb|		8 |r%   