
    oiP              	           S SK r S SKJrJrJr  S SKrS SKJr  S SKJs  J	r
  S SKJr   " S S\R                  \5      r " S S\5      r " S S	\5      r " S
 S\5      rSS\S\S\\\4   4S jjrS rSS jrg)    N)AnyOptionalUnion)LycorisLayerc                   F  ^  \ rS rSrSrS\R                  SS4U 4S jjr\S\	\
   4S j5       rS\
S	\S
\S\S\4
S jrS\
4S jrS\
4S jrS r SS\
S	\S\S\S\S\S\S\S\S\SS4S jjrS\
S\R*                  4S jrS\R*                  S\R*                  4S jrSrU =r$ )	LoKrLayer   lokr_w1	lokr_w1_a	lokr_w1_blokr_w2	lokr_w2_a	lokr_w2_blokr_t2
base_layerreturnNc                   > [         TU ]  5         [        R                  " X5        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l	        [        R                  " 0 5      U l
        [        R                  " 0 5      U l        g N)super__init__r   nnParameterDictr   r   r   r   r   r   r   )selfr   	__class__s     P/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/lokr/layer.pyr   LoKrLayer.__init__&   s    d/ ''+))"-))"-''+))"-))"-''+    c                     1 U R                   kU R                  kU R                  kU R                  kU R                  kU R
                  kU R                  k$ r   r
   )r   s    r   _available_adaptersLoKrLayer._available_adapters3   sj    
\\
^^
 ^^
 \\	

 ^^
 ^^
 \\
 	
r   adapter_nameruse_w1use_w2use_effective_conv2dc           	      	   U(       aE  [         R                  " [        R                  " US   S   US   S   5      5      U R                  U'   O{[         R                  " [        R                  " US   S   U5      5      U R
                  U'   [         R                  " [        R                  " X#S   S   5      5      U R                  U'   [        U5      S:X  Ga  U(       aI  [         R                  " [        R                  " US   S   US   S   /USS  Q76 5      U R                  U'   g U(       a  [         R                  " [        R                  " X"US   US   5      5      U R                  U'   [         R                  " [        R                  " X#S   S   5      5      U R                  U'   [         R                  " [        R                  " X#S   S   5      5      U R                  U'   g [         R                  " [        R                  " US   S   U5      5      U R                  U'   [         R                  " [        R                  " X#S   S   US   -  US   -  5      5      U R                  U'   g [        U5      S:X  Ga  U(       aI  [         R                  " [        R                  " US   S   US   S   US   5      5      U R                  U'   g U(       a  [         R                  " [        R                  " X"US   S5      5      U R                  U'   [         R                  " [        R                  " X#S   S   5      5      U R                  U'   [         R                  " [        R                  " X#S   S   5      5      U R                  U'   g [         R                  " [        R                  " US   S   U5      5      U R                  U'   [         R                  " [        R                  " X#S   S   US   -  5      5      U R                  U'   g U(       aE  [         R                  " [        R                  " US   S   US   S   5      5      U R                  U'   g [         R                  " [        R                  " US   S   U5      5      U R                  U'   [         R                  " [        R                  " X#S   S   5      5      U R                  U'   g )Nr               )r   	Parametertorchemptyr   r   r   lenr   r   r   r   )r   r"   r#   shaper$   r%   r&   s          r   create_adapter_parameters#LoKrLayer.create_adapter_parameters?   s    )+ekk%(1+uUVxXY{6[)\DLL&+-<<E!HQKQR8S+TDNN<(+-<<AQxPQ{8S+TDNN<( u:?-/\\%++eAhqkSXYZS[\]S^:kafghgiaj:k-l\*%-/\\%++aERSHV[\]V^:_-`\*/1||EKKQR8TU;<W/X|,/1||EKKQR8TU;<W/X|,/1||EKKaQRUV<W/X|,/1||EKKQR8TU;Y^_`YaKadijkdlKl<m/n|,Z1_-/\\%++eAhqkSXYZS[\]S^`efg`h:i-j\*% .0\\%++aERSHVW:X-Y\*/1||EKKQR8TU;<W/X|,/1||EKKQR8TU;<W/X|,/1||EKKaQRUV<W/X|,/1||EKKQR8TU;Y^_`YaKa<b/c|, -/\\%++eAhqkSXYZS[\]S^:_-`\*/1||EKKaQRUV<W/X|,/1||EKKQR8TU;<W/X|,r   c                    XR                   ;   a-  [        R                  R                  U R                   U   5        Ok[        R                  R                  U R                  U   5        [        R                  R                  U R                  U   [        R                  " S5      S9  XR                  ;   a@  [        R                  R                  U R                  U   [        R                  " S5      S9  O~[        R                  R                  U R                  U   [        R                  " S5      S9  [        R                  R                  U R                  U   [        R                  " S5      S9  XR                  ;   a@  [        R                  R                  U R                  U   [        R                  " S5      S9  g g N   )a)r   r   initzeros_r   kaiming_uniform_r   mathsqrtr   r   r   r   r   r"   s     r   reset_adapter_parameters"LoKrLayer.reset_adapter_parametersm   s   <<'GGNN4<<56GGNN4>>,78GG$$T^^L%ATYYq\$R<<'GG$$T\\,%?499Q<$PGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R<<'GG$$T\\,%?499Q<$P (r   c                    XR                   ;   a@  [        R                  R                  U R                   U   [        R
                  " S5      S9  O~[        R                  R                  U R                  U   [        R
                  " S5      S9  [        R                  R                  U R                  U   [        R
                  " S5      S9  XR                  ;   a@  [        R                  R                  U R                  U   [        R
                  " S5      S9  O~[        R                  R                  U R                  U   [        R
                  " S5      S9  [        R                  R                  U R                  U   [        R
                  " S5      S9  XR                  ;   a@  [        R                  R                  U R                  U   [        R
                  " S5      S9  g g r4   )r   r   r7   r9   r:   r;   r   r   r   r   r   r   r<   s     r   reset_adapter_parameters_random)LoKrLayer.reset_adapter_parameters_random}   s=   <<'GG$$T\\,%?499Q<$PGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R<<'GG$$T\\,%?499Q<$PGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R<<'GG$$T\\,%?499Q<$P (r   c                    XR                   ;   a@  [        R                  R                  U R                   U   [        R
                  " S5      S9  O~[        R                  R                  U R                  U   [        R
                  " S5      S9  [        R                  R                  U R                  U   [        R
                  " S5      S9  XR                  ;   a-  [        R                  R                  U R                  U   5        Ok[        R                  R                  U R                  U   5        [        R                  R                  U R                  U   [        R
                  " S5      S9  XR                  ;   a@  [        R                  R                  U R                  U   [        R
                  " S5      S9  g g r4   )r   r   r7   r9   r:   r;   r   r   r   r8   r   r   r   r<   s     r   $reset_adapter_parameters_lycoris_way.LoKrLayer.reset_adapter_parameters_lycoris_way   s   <<'GG$$T\\,%?499Q<$PGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R<<'GGNN4<<56GGNN4>>,78GG$$T^^L%ATYYq\$R<<'GG$$T\\,%?499Q<$P (r   alpharank_dropoutmodule_dropoutinit_weightsdecompose_bothdecompose_factorinference_modec                    US::  a  [        SU 35      eX R                  U'   X0R                  U'   X2-  U R                  U'   X@R                  U'   XPR
                  U'   US   U R                  U'   U R                  5       n[        U[        R                  5      (       a  UR                  UR                  p[        X5      u  nn[        X5      u  nnUU4UU44nU=(       a    U[        US   S   US   S   5      S-  :  (       + nU[        US   S   US   S   5      S-  :  + nSnGO[        U[        R                  5      (       a  UR                   UR"                  pUR$                  n[        X5      u  nn[        X5      u  nnUU4UU4/UQ7nU=(       a    U[        US   S   US   S   5      S-  :  (       + nU[        US   S   US   S   5      S-  :  nU=(       a    UR$                  S:g  nO[        U[        R&                  5      (       a  UR                   UR"                  pUR$                  S   4n[        X5      u  nn[        X5      u  nnUU4UU4/UQ7nU=(       a    U[        US   S   US   S   5      S-  :  (       + nU[        US   S   US   S   5      S-  :  nU=(       a    UR$                  S   S:g  nO![)        S[+        U5      R,                   35      eU R/                  XUUUU5        U(       a*  US	:X  a  U R1                  U5        O#U R3                  U5        OU R5                  U5        U R7                  U5        U R9                  U R:                  U
S
9  g)a  Internal function to create lokr adapter

Args:
    adapter_name (`str`): Name for the adapter to add.
    r (`int`): Rank for the added adapter.
    alpha (`float`): Alpha for the added adapter.
    rank_dropout (`float`): The dropout probability for rank dimension during training
    module_dropout (`float`): The dropout probability for disabling adapter during training.
    init_weights (`bool`): Whether to initialize adapter weights.
    use_effective_conv2d (`bool`): Use parameter effective decomposition for Conv2d with ksize > 1.
    decompose_both (`bool`): Perform rank decomposition of left kronecker product matrix.
    decompose_factor (`int`): Kronecker product decomposition factor.
r   z?`r` should be a positive integer value but the value passed is rank_dropout_scaler(   r*   F)r(   r(   z0LoKr is not implemented for base layers of type lycoris)rK   N)
ValueErrorr#   rE   scalingrF   rG   rM   get_base_layer
isinstancer   Linearin_featuresout_featuresfactorizationmaxConv2din_channelsout_channelskernel_sizeConv1d	TypeErrortype__name__r1   rC   r=   r@   %_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r   r"   r#   rE   rF   rG   rH   r&   rI   rJ   rK   kwargsr   in_dimout_dimin_min_nout_lout_kr0   r$   r%   k_sizes                          r   update_layerLoKrLayer.update_layer   s~   6 6^_`^abcc |#(

< %*Y\"*6,',:L)067K0L-((*
 j")),,(44j6M6MG&v@JD$(CLE5U^dD\2E(RQU1Xa[%(1+1NQR1R-RSFc%(1+uQx{;a?@F#( 
BII..(44j6M6MG++F&v@JD$(CLE5U^dD\;F;E(RQU1Xa[%(1+1NQR1R-RSF#eAhqk58A;7!;;F $8#\J<R<RV\<\ 
BII..(44j6M6MG ,,Q/1F&v@JD$(CLE5U^dD\;F;E(RQU1Xa[%(1+1NQR1R-RSF#eAhqk58A;7!;;F
 $8#ZJ<R<RST<UYZ<Z NtT^O_OhOhNijkk 	&&|vvOcd y(99,G--l;00> 	22<@--nMr   c           	         XR                   ;   a  U R                   U   nOU R                  U   U R                  U   -  nXR                  ;   a  U R                  U   nOcXR                  ;   a5  [        U R                  U   U R                  U   U R                  U   5      nOU R                  U   U R                  U   -  n[        X#U R                  U   5      nU R                  5       nUR                  UR                  R                  5      nU R                  U   nU R                  (       a  U(       a  [         R"                  " UR%                  S5      5      U:  R'                  5       nUR(                  " S/S/[+        UR                  SS  5      -  Q76 R-                  UR.                  5      nU R0                  U   (       a  XwR3                  5       -  nXG-  nU$ )Nr   r(   )r   r   r   r   r   make_weight_cpr   r   	make_kronrP   rQ   reshapeweightr0   rF   trainingr-   randsizefloatviewr/   todevicerM   mean)r   r"   w1w2rr   r   rF   drops           r   get_delta_weightLoKrLayer.get_delta_weight  s   <<'l+B-|0LLB<<'l+B\\)\ :DNN<<XZ^ZhZhiuZvwB-|0LLB 24<<#=> ((*
 
 1 1 7 78 ((6==\JJv{{1~.=DDFD99R>1#FLL,<(="=>AA&--PD&&|4		#NFr   xc                 d   UR                   nU R                  (       a8  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nOU R                  (       a  U R                  " U/UQ70 UD6nOU R                  " U/UQ70 UD6nU R
                   H{  nX`R                  ;  a  M  U R                  U   nU R                  (       a/  U R                  (       d  MG  [        R                  " S5      U:  d  Mc  XPR                  " Xa/UQ70 UD6-   nM}     UR                  U5      nU$ )Nr(   )dtypedisable_adaptersmergedunmerger   rb   r    rG   rs   r-   rt   _get_delta_activationsrx   )r   r   argsrc   previous_dtyperesultactive_adapterrG   s           r   forwardLoKrLayer.forward#  s      {{__Q888F[[__Q888F__Q888F #'"6"6!)A)AA!%!4!4^!D 4===UZZ]^=[#&A&A.&eVZ&e^d&eeF #7 >*r   )r   r   r   r   r   r   r   )F)r_   
__module____qualname____firstlineno__adapter_layer_namesr   Moduler   propertysetstrr    intboolr1   r=   r@   rC   rv   rk   r-   Tensorr~   r   __static_attributes____classcell__r   s   @r   r   r      sf   ,299 , , 	
SX 	
 	
,Y,Y ,Y
 ,Y ,Y #,Y\QS Q QC Q"Q6  %aNaN aN 	aN
 aN aN aN #aN aN aN aN 
aNF S  U\\  D 5<<  r   r   c                     ^  \ rS rSrSr        SS\R                  S\\\	\
R                  4      S\\
R                     S\	S\S\S	\S
\S\4U 4S jjjrS\	S\
R"                  S\S\S\
R"                  4
S jrS\	4U 4S jjrSrU =r$ )rS   i>  z LoKr implemented in Linear layerr   ry   r   r"   r#   rE   rF   rG   rH   c
                 Z   > [         TU ]  U5        X@l        U R                  " XEXgX40 U
D6  g r   r   r   _active_adapterrk   )r   r   ry   r   r"   r#   rE   rF   rG   rH   rc   r   s              r   r   Linear.__init__A  s2     	$  ,,5g`fgr   inputr   rc   r   c                     U R                  U5      nU R                  X%R                  5      n[        R                  " X%5      $ r   )r~   _cast_input_dtyper   Flinear)r   r"   r   r   rc   delta_weights         r   r   Linear._get_delta_activationsT  s9     ,,\:&&u.@.@Axx,,r   c                 *   > [         TU ]  5       nSU-   $ Nzlokr.r   __repr__r   repr   s     r   r   Linear.__repr__\      g }r   r   )NNdefaultr           r   r   Tr_   r   r   r   __doc__r   r   r   r   r   r-   ry   r   r   rv   r   r   r   r   r   r   r   r   r   s   @r   rS   rS   >  s    *
 6:'+%! #!hIIh sELL012h $	h
 h h h h h h h&--(--=@-LO-	-#  r   rS   c                     ^  \ rS rSrSr         SS\R                  S\\\	\
R                  4      S\\
R                     S\	S\S\S	\S
\S\S\4U 4S jjjrS\	S\
R"                  S\S\S\
R"                  4
S jrS\	4U 4S jjrSrU =r$ )rX   ia  z LoKr implemented in Conv2d layerr   ry   r   r"   r#   rE   rF   rG   r&   rH   c           	      \   > [         TU ]  U5        X@l        U R                  " XEXgXU	40 UD6  g r   r   r   r   ry   r   r"   r#   rE   rF   rG   r&   rH   rc   r   s               r   r   Conv2d.__init__d  :     	$  ,U.Pd	
hn	
r   r   r   rc   r   c           	          U R                  U5      nU R                  X%R                  5      nU R                  5       n[        R
                  " UUUR                  UR                  UR                  UR                  S9$ N)stridepaddingdilationgroups)
r~   r   r   rQ   r   conv2dr   r   r   r   r   r"   r   r   rc   r   r   s          r   r   Conv2d._get_delta_activationsz  s     ,,\:&&u.@.@A((*
xx$$&&(($$
 	
r   c                 *   > [         TU ]  5       nSU-   $ r   r   r   s     r   r   Conv2d.__repr__  r   r   r   	NNr   r   r   r   r   FTr   r   s   @r   rX   rX   a      *
 6:'+%! #%*!
II
 sELL012
 $	

 
 
 
 
 
 #
 
 
,

(-
=@
LO
	
 #  r   rX   c                     ^  \ rS rSrSr         SS\R                  S\\\	\
R                  4      S\\
R                     S\	S\S\S	\S
\S\S\4U 4S jjjrS\	S\
R"                  S\S\S\
R"                  4
S jrS\	4U 4S jjrSrU =r$ )r\   i  z LoKr implemented in Conv1d layerr   ry   r   r"   r#   rE   rF   rG   r&   rH   c           	      \   > [         TU ]  U5        X@l        U R                  " XEXgXU	40 UD6  g r   r   r   s               r   r   Conv1d.__init__  r   r   r   r   rc   r   c           	          U R                  U5      nU R                  X%R                  5      nU R                  5       n[        R
                  " UUUR                  UR                  UR                  UR                  S9$ r   )
r~   r   r   rQ   r   conv1dr   r   r   r   r   s          r   r   Conv1d._get_delta_activations  r   r   c                 *   > [         TU ]  5       nSU-   $ r   r   r   s     r   r   Conv1d.__repr__  r   r   r   r   r   r   s   @r   r\   r\     r   r   r\   	dimensionfactorr   c                     US:  a  X-  S:X  a	  UnX-  nX#4$ US:X  a  U nSU p2X#-   nX#:  a7  US-   nX-  S:w  a  US-  nX-  S:w  a  M  X-  nXV-   U:  d  XQ:  a  O	XVp2X#:  a  M7  X#:  a  X#p#X#4$ )aN  Factorizes the provided number into the product of two numbers

Args:
    dimension (`int`): The number that needs to be factorized.
    factor (`int`, optional):
        Factorization divider. The algorithm will try to output two numbers, one of each will be as close to the
        factor as possible. If -1 is provided, the decomposition algorithm would try to search dividers near the
        square root of the dimension. Defaults to -1.

Returns:
    Tuple[`int`, `int`]: A tuple of two numbers, whose product is equal to the provided number. The first number is
    always less than or equal to the second.

Example:
    ```py
    >>> factorization(256, factor=-1)
    (16, 16)

    >>> factorization(128, factor=-1)
    (8, 16)

    >>> factorization(127, factor=-1)
    (1, 127)

    >>> factorization(128, factor=4)
    (4, 32)
    ```
r   rn   r(    )r   r   mnlengthnew_mnew_ns          r   rV   rV     s    < zy)a/t|iqUF
%A1$QJE 1$"=6!U^q % 	u14Kr   c                 6    [         R                  " SXU5      nU$ )Nzi j k l, i p, j r -> p r k l)r-   einsum)twawbrebuild2s       r   ro   ro     s    ||:A2FHOr   c                     [        UR                  5      S:X  a   U R                  S5      R                  S5      n UR                  5       n[        R
                  " X5      nX2-  $ )Nr)   r*   )r/   r0   	unsqueeze
contiguousr-   kron)r{   r|   scalerebuilds       r   rp   rp     sK    
288}\\!_&&q)	Bjj G?r   )rn   )g      ?)r:   typingr   r   r   r-   torch.nnr   torch.nn.functional
functionalr   peft.tuners.lycoris_utilsr   r   r   rS   rX   r\   r   tuplerV   ro   rp   r   r   r   <module>r      s     ' '     2b		< bJ	 Y  F+Y +\+Y +b1S 1# 1uS#X 1h
r   