
    oiI                     @   S SK r S SKJr  S SKrS SKJr  S SKJs  Jr  S SK	J
r
   " S S\R                  \
5      r " S S\5      r " S S	\5      r " S
 S\5      r " S S\R                   R"                  5      r " S S\R                   R"                  5      rS rS rg)    N)Any)LycorisLayerc                   8  ^  \ rS rSrSrS\R                  4U 4S jjr\S\	\
   4S j5       rS\
S\S	\\S
4   4S jrS\
4S jrS\
4S jr  SS\
S\S\S\S\S\S\S\SS4S jjrS\
S\R*                  4S jrS\R*                  S\R*                  4S jrSrU =r$ )	LoHaLayer   	hada_w1_a	hada_w1_b	hada_w2_a	hada_w2_bhada_t1hada_t2
base_layerc                   > [         TU ]  5         [        R                  " X5        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l	        [        R                  " 0 5      U l
        g N)super__init__r   nnParameterDictr	   r
   r   r   r   r   )selfr   	__class__s     P/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/loha/layer.pyr   LoHaLayer.__init__   s    d/ ))"-))"-))"-))"-''+''+    returnc                     1 U R                   kU R                  kU R                  kU R                  kU R                  kU R
                  k$ r   r   )r   s    r   _available_adaptersLoHaLayer._available_adapters*   sD    qq$..q4>>qDNNqUYUaUaqdhdpdpqqr   adapter_namershape.c           	         [        U5      S:X  Gag  [        R                  " [        R                  " X"US   US   5      5      U R
                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   [        R                  " [        R                  " X"US   US   5      5      U R                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   g [        U5      S:X  Gaa  [        R                  " [        R                  " X"US   S5      5      U R
                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   [        R                  " [        R                  " X"US   S5      5      U R                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   g [        R                  " [        R                  " US   U5      5      U R                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   [        R                  " [        R                  " US   U5      5      U R                  U'   [        R                  " [        R                  " X#S   5      5      U R                  U'   g )N         r      )lenr   	Parametertorchemptyr   r	   r
   r   r   r   )r   r   r    r!   s       r   create_adapter_parameters#LoHaLayer.create_adapter_parameters.   s|   u:?)+ekk!aRWXYRZ6[)\DLL&+-<<AQx8P+QDNN<(+-<<AQx8P+QDNN<()+ekk!aRWXYRZ6[)\DLL&+-<<AQx8P+QDNN<(+-<<AQx8P+QDNN<(Z1_)+ekk!aRS6T)UDLL&+-<<AQx8P+QDNN<(+-<<AQx8P+QDNN<()+ekk!aRS6T)UDLL&+-<<AQx8P+QDNN<(+-<<AQx8P+QDNN<(+-<<E!Ha8P+QDNN<(+-<<AQx8P+QDNN<(+-<<E!Ha8P+QDNN<(+-<<AQx8P+QDNN<(r   c                 H   XR                   R                  5       ;   a  [        R                  R	                  U R                   U   [
        R                  " S5      S9  [        R                  R	                  U R                  U   [
        R                  " S5      S9  [        R                  R	                  U R                  U   [
        R                  " S5      S9  [        R                  R                  U R                  U   5        XR                  R                  5       ;   a  [        R                  R	                  U R                  U   [
        R                  " S5      S9  [        R                  R	                  U R                  U   [
        R                  " S5      S9  g g N   )a)r	   keysr   initkaiming_uniform_mathsqrtr
   r   zeros_r   r   r   r   r   s     r   reset_adapter_parameters"LoHaLayer.reset_adapter_parametersG   s    >>..00GG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$RGGNN4>>,78<<,,..GG$$T\\,%?499Q<$PGG$$T\\,%?499Q<$P /r   c                 n   XR                   R                  5       ;   a  [        R                  R	                  U R                   U   [
        R                  " S5      S9  [        R                  R	                  U R                  U   [
        R                  " S5      S9  [        R                  R	                  U R                  U   [
        R                  " S5      S9  [        R                  R	                  U R                  U   [
        R                  " S5      S9  XR                  R                  5       ;   a  [        R                  R	                  U R                  U   [
        R                  " S5      S9  [        R                  R	                  U R                  U   [
        R                  " S5      S9  g g r.   )r	   r1   r   r2   r3   r4   r5   r
   r   r   r   r   r7   s     r   reset_adapter_parameters_random)LoHaLayer.reset_adapter_parameters_randomV   s    >>..00GG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R<<,,..GG$$T\\,%?499Q<$PGG$$T\\,%?499Q<$P /r   alpharank_dropoutmodule_dropoutinit_weightsuse_effective_conv2dinference_modeNc	                    US::  a  [        SU 35      eX R                  U'   X0R                  U'   X2-  U R                  U'   X@R                  U'   XPR
                  U'   U R                  5       n
[        U
[        R                  5      (       a!  [        U
R                  R                  5      nGOO[        U
[        R                  5      (       a~  U=(       a    U
R                  S:g  nU(       a&  U
R                  U
R                   /U
R                  Q7nOU
R                  U
R                   U
R                  S   -  U
R                  S   -  4nO[        U
[        R"                  5      (       ar  U=(       a    U
R                  S   S:g  nU(       a'  U
R                  U
R                   U
R                  S   4nOJU
R                  U
R                   U
R                  S   -  4nO![%        S['        U
5      R(                   35      eU R+                  XU5        U(       a  U R-                  U5        OU R/                  U5        U R1                  U5        U R3                  U R4                  US9  g)a@  Internal function to create loha adapter

Args:
    adapter_name (`str`): Name for the adapter to add.
    r (`int`): Rank for the added adapter.
    alpha (`float`): Alpha for the added adapter.
    rank_dropout (`float`): The dropout probability for rank dimension during training.
    module_dropout (`float`): The dropout probability for disabling adapter during training.
    init_weights (`bool`): Whether to initialize weights.
    use_effective_conv2d (`bool`, *optional*, defaults to `False`):
        Use parameter effective decomposition for Conv2d with ksize > 1.
r   z?`r` should be a positive integer value but the value passed is )r&   r&   r&   z0LoHa is not implemented for base layers of type )rB   N)
ValueErrorr    r=   scalingr>   r?   get_base_layer
isinstancer   Lineartupleweightr!   Conv2dkernel_sizeout_channelsin_channelsConv1d	TypeErrortype__name__r+   r8   r;   %_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r   r   r    r=   r>   r?   r@   rA   rB   kwargsr   r!   s               r   update_layerLoHaLayer.update_layere   s'   0 6^_`^abcc |#(

< %*Y\"*6,',:L) ((*
j")),,*++112E
BII.. $8#\J<R<RV\<\ ##00*2H2Hb:KaKab ++**Z-C-CA-FFI_I_`aIbb 
BII..
 $8#ZJ<R<RST<UYZ<Z ##00*2H2H*J`J`abJcd ++**Z-C-CA-FF
 NtT^O_OhOhNijkk 	&&|> )),700> 	22<@--nMr   c                    XR                   R                  5       ;   a  [        U R                   U   U R                  U   U R                  U   U R
                  U   U R                  U   U R                  U   [        R                  " U R                  U   5      S9nOb[        U R                  U   U R                  U   U R                  U   U R                  U   [        R                  " U R                  U   5      S9nU R                  5       nUR                  UR                  R                  5      nU R                   U   nU R"                  (       a  U(       a  [        R$                  " UR'                  S5      5      U:  R)                  UR*                  5      nUR,                  " S/S/[/        UR                  SS  5      -  Q76 R)                  UR0                  5      nXUR3                  5       -  nX%-  nU$ )N)scaler   r&   )r   r1   make_weight_cpr	   r
   r   r   r   r)   tensorrE   make_weightrF   reshaperJ   r!   r>   trainingrandsizetodtypeviewr'   devicemean)r   r   rJ   r   r>   drops         r   get_delta_weightLoHaLayer.get_delta_weight   s   <<,,..#\*|,|,\*|,|,ll4<<#=>F !|,|,|,|,ll4<<#=>F ((*
 
 1 1 7 78 ((6==\JJv{{1~.=AA&,,OD99R>1#FLL,<(="=>AA&--PD IIKDNFr   xc                 d   UR                   nU R                  (       a8  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nOU R                  (       a  U R                  " U/UQ70 UD6nOU R                  " U/UQ70 UD6nU R
                   H{  nX`R                  ;  a  M  U R                  U   nU R                  (       a/  U R                  (       d  MG  [        R                  " S5      U:  d  Mc  XPR                  " Xa/UQ70 UD6-   nM}     UR                  U5      nU$ )Nr&   )rd   disable_adaptersmergedunmerger   rU   r   r?   r`   r)   ra   _get_delta_activationsrc   )r   rk   argsrV   previous_dtyperesultactive_adapterr?   s           r   forwardLoHaLayer.forward   s      {{__Q888F[[__Q888F__Q888F #'"6"6!)A)AA!%!4!4^!D 4===UZZ]^=[#&A&A.&eVZ&e^d&eeF #7 >*r   )r   r   r	   r
   r   r   )FF)rR   
__module____qualname____firstlineno__adapter_layer_namesr   Moduler   propertysetstrr   intrI   r+   r8   r;   floatboolrW   r)   Tensorri   ru   __static_attributes____classcell__r   s   @r   r   r      s2   d
,299 
, rSX r rRc Rc R%PSUXPX/ R2QS QQC Q. &+$NNNN NN 	NN
 NN NN NN #NN NN 
NN`%S %U\\ %N 5<<  r   r   c                      ^  \ rS rSrSr      SS\R                  S\S\S\	S\	S\	S	\
4U 4S
 jjjrS\S\R                  S\S\S\R                  4
S jrS\4U 4S jjrSrU =r$ )rH      z LoHa implemented in Linear layerr   r   r    r=   r>   r?   r@   c                 Z   > [         T	U ]  U5        X l        U R                  " X#XEXg40 UD6  g r   r   r   _active_adapterrW   )
r   r   r   r    r=   r>   r?   r@   rV   r   s
            r   r   Linear.__init__   s2     	$  ,,5g`fgr   inputrq   rV   r   c                     U R                  U5      nU R                  X%R                  5      n[        R                  " X%5      $ r   )ri   _cast_input_dtyperd   Flinear)r   r   r   rq   rV   delta_weights         r   rp   Linear._get_delta_activations  s9     ,,\:&&u.@.@Axx,,r   c                 *   > [         TU ]  5       nSU-   $ Nzloha.r   __repr__r   repr   s     r   r   Linear.__repr__      g }r   r   )defaultr           r   r   TrR   rw   rx   ry   __doc__r   r{   r~   r   r   r   r   r)   r   r   rp   r   r   r   r   s   @r   rH   rH      s    *
 &! #!hIIh h 	h
 h h h h h"--(--=@-LO-	-#  r   rH   c                      ^  \ rS rSrSr       SS\R                  S\S\S\	S\	S\	S	\
S
\
4U 4S jjjrS\S\R                  S\S\S\R                  4
S jrS\4U 4S jjrSrU =r$ )rK   i  z LoHa implemented in Conv2d layerr   r   r    r=   r>   r?   rA   r@   c	           	      \   > [         T
U ]  U5        X l        U R                  " X#XEXhU40 U	D6  g r   r   r   r   r   r    r=   r>   r?   rA   r@   rV   r   s             r   r   Conv2d.__init__  :     	$  ,U.Pd	
hn	
r   r   rq   rV   r   c           	          U R                  U5      nU R                  X%R                  5      nU R                  5       n[        R
                  " UUUR                  UR                  UR                  UR                  S9$ N)stridepaddingdilationgroups)
ri   r   rd   rF   r   conv2dr   r   r   r   r   r   r   rq   rV   r   r   s          r   rp   Conv2d._get_delta_activations/  s     ,,\:&&u.@.@A((*
xx$$&&(($$
 	
r   c                 *   > [         TU ]  5       nSU-   $ r   r   r   s     r   r   Conv2d.__repr__?  r   r   r   r   r   r   r   r   FTr   r   s   @r   rK   rK         *
 &! #%*!
II
 
 	

 
 
 
 #
 
 
(

(-
=@
LO
	
 #  r   rK   c                      ^  \ rS rSrSr       SS\R                  S\S\S\	S\	S\	S	\
S
\
4U 4S jjjrS\S\R                  S\S\S\R                  4
S jrS\4U 4S jjrSrU =r$ )rO   iD  z LoHa implemented in Conv1d layerr   r   r    r=   r>   r?   rA   r@   c	           	      \   > [         T
U ]  U5        X l        U R                  " X#XEXhU40 U	D6  g r   r   r   s             r   r   Conv1d.__init__G  r   r   r   rq   rV   r   c           	          U R                  U5      nU R                  X%R                  5      nU R                  5       n[        R
                  " UUUR                  UR                  UR                  UR                  S9$ r   )
ri   r   rd   rF   r   conv1dr   r   r   r   r   s          r   rp   Conv1d._get_delta_activations[  r   r   c                 *   > [         TU ]  5       nSU-   $ r   r   r   s     r   r   Conv1d.__repr__k  r   r   r   r   r   r   s   @r   rO   rO   D  r   r   rO   c                   Z    \ rS rSr\\R                  " S5      4S j5       r\S 5       rSr	g)
HadaWeightis  r&   c                 D    U R                  XX4U5        X-  X4-  -  U-  nU$ r   )save_for_backward)ctxw1aw1bw2aw2brZ   diff_weights          r   ru   HadaWeight.forwardt  s-    c%8	ci0E9r   c                     U R                   u  p#pEnX-  nXU-  -  nXsR                  -  nUR                  U-  n	XU-  -  nXuR                  -  n
UR                  U-  nAXXS 4$ r   )saved_tensorsT)r   grad_outr   r   r   r   rZ   tempgrad_w1agrad_w1bgrad_w2agrad_w2bs               r   backwardHadaWeight.backwardz  st    &)&7&7#3U#9%%%<554<9%%%<554<8t;;r    N
rR   rw   rx   ry   staticmethodr)   r]   ru   r   r   r   r   r   r   r   s  s3    /4||A  
 < <r   r   c                   Z    \ rS rSr\\R                  " S5      4S j5       r\S 5       rSr	g)HadaWeightCPi  r&   c           	          U R                  XX4XVU5        [        R                  " SXU5      n[        R                  " SXFU5      n	X-  U-  $ )Nzi j k l, j r, i p -> p r k l)r   r)   einsum)
r   t1r   r   t2r   r   rZ   rebuild1rebuild2s
             r   ru   HadaWeightCP.forward  sJ    bs%@<< >M<< >M"U**r   c                    U R                   u  p#pEpgnX-  n[        R                  " SXW5      n	[        R                  " SX5      n
X-  nA
[        R                  " SX5      n[        R                  " SXR                  5      nAA	[        R                  " SX-5      n[        R                  " SXR                  5      nA[        R                  " SX$5      n	[        R                  " SX5      n
X-  nA
[        R                  " SX5      n[        R                  " SXR                  5      nAA	[        R                  " SX]5      n[        R                  " SXR                  5      nAXUUUUS 4$ )Nzi j k l, j r -> i r k lzi j k l, i r -> r j k lzr j k l, i j k l -> r izi r k l, i j k l -> r j)r   r)   r   r   )r   r   r   r   r   r   r   r   rZ   r   rebuildgrad_wr   	grad_tempr   grad_t1r   r   grad_t2s                      r   r   HadaWeightCP.backward  s3   .1.?.?+#3U#||5r?,,8$D#<< 94HLL!:FEEJ	D<< 92I,,8)UUK||5r?,,8$D#<< 94HLL!:FEEJ	D<< 92I,,8)UUK(GXxMMr   r   Nr   r   r   r   r   r     s5    7<||A + + N Nr   r   c                 0    [         R                  XX#U5      $ r   )r   apply)r   r   r   r   rZ   s        r   r^   r^     s    Cc66r   c           	      2    [         R                  XX#XEU5      $ r   )r   r   )r   r   r   r   r   r   rZ   s          r   r\   r\     s    bs%@@r   )r4   typingr   r)   torch.nnr   torch.nn.functional
functionalr   peft.tuners.lycoris_utilsr   r{   r   rH   rK   rO   autogradFunctionr   r   r^   r\   r   r   r   <module>r      s          2[		< [|Y B)Y )X)Y )^<(( <.*N5>>** *NZ7Ar   