
    oi !                         S SK Jr  S SKrS SKJs  Jr  S SKJr  S SKJrJ	r	  S SK
Jr   " S S\R                  5      r " S S	\5      r " S
 S\5      r " S S\5      r " S S\5      r " S S\5      rg)    )deepcopyN)nn)dequantize_module_weightgather_params_ctx)	transposec                   ~   ^  \ rS rSrU 4S jrS\R                  4S jrSS.SS jjrSS	.S
 jr	S\
4U 4S jjrSrU =r$ )DoraLinearLayer   c                 .   > [         TU ]  5         Xl        g N)super__init__fan_in_fan_outselfr   	__class__s     O/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/lora/dora.pyr   DoraLinearLayer.__init__   s    ,    returnc                     [        XR                  5      nXU-  -   n[        R                  R	                  USS9R                  UR                  5      nU$ )N   )dim)r   r   torchlinalgnormtodtype)r   weightlora_weightscalingweight_norms        r   get_weight_normDoraLinearLayer.get_weight_norm   sM    6#6#67K//ll''A'699&,,Gr   F)place_on_cpuNc                >   UR                   [        R                  :H  nU(       a   UR                  5       nUR                  5       n[	        UR                  5       5         UR                  R                  S:X  a  [        U5      n[        U5      nUR                  R                  S:  ad  UR                  S   n[        R                  " UR                  SU/5      UR                  US/5      5      n	U	R                  UR                  5      n	OX2-  n	U(       a  U	R!                  5       n	U R#                  UR%                  UR&                  5      X5      n
S S S 5        U(       a  W
R%                  S5      n
[(        R*                  " W
SS9U l        g ! , (       d  f       NA= f)N
Linear4bit   r   cpuT)requires_grad)r   r   float16floatr   
parametersr   __name__r   r   datandimshapemmviewreshapehalfr#   r   devicer   	Parameterr   )r   
base_layerlora_Alora_Br!   r%   dtype_is_fp16r   rr    r"   s              r   update_layerDoraLinearLayer.update_layer%   s/   5\\^F\\^Fz4467##,,< &j1
-j9F{{1$LLO#hhv{{B7';V[[!R=QR)11&,,?$o)..0..vyy/G^K! 8$ %../Kll;dC) 87s   C5F
Fbase_resultc                z   [         R                  " UR                  R                  S   UR                  R                  UR
                  S9nU" U" U5      5      R                  nU R                  n	[        U5      n
U
R                  UR
                  5      n
U R                  XR                  5       U5      nUR                  5       nX-  R                  SS5      nU" U" U5      5      nSnUb  UR                  nUb  Xn-
  nO*[        R                  " U[        XR                   5      5      nUS-
  U-  X-  U-  -   nU$ )y
For DoRA, calculate the extra output from LoRA with DoRA applied. This should be added on top of the base layer
output.
r   )r7   r   r)   N)r   eyer   r2   r7   r   Tr   r   r#   detachr4   biasFlinearr   r   )r   xr:   r;   r!   r9   rA   x_eyer    	magnituder   r"   mag_norm_scalelora_resultrG   result_doras                   r   forwardDoraLinearLayer.forwardB   s    		&----a09M9MUVU\U\]VE]+--KK	)*5177#**63E3E3GQ "((*#1772>VAY'"??D)0((1i8K8K&LMK%)[8>;WZa;aar   c                 *   > [         TU ]  5       nSU-   $ Nz
lora.dora.r   __repr__r   repr   s     r   rU   DoraLinearLayer.__repr__g       g c!!r   )r   r   )r   N)r/   
__module____qualname____firstlineno__r   r   Tensorr#   r>   rP   strrU   __static_attributes____classcell__r   s   @r   r	   r	      sC    -u||  QV D: NR #J"# " "r   r	   c                   6   ^  \ rS rSrS rS\4U 4S jjrSrU =r$ )DoraEmbeddingLayerl   c                    X#-  R                   nU R                  nUR                  n	U R                  XR                  5       U5      n
U
R                  5       n
X-  nX" X5      U-  -  U-  nX4$ )rC   )rE   r   r#   rF   )r   rJ   r:   r;   r!   r9   embed_fnr    rL   r   r"   rM   rO   s                r   rP   DoraEmbeddingLayer.forwardm   sz    
 ))KK	""**63E3E3GQ "((*"0$(;f(DEO**r   r   c                 *   > [         TU ]  5       nSU-   $ rS   rT   rV   s     r   rU   DoraEmbeddingLayer.__repr__   rY   r    )	r/   rZ   r[   r\   rP   r^   rU   r_   r`   ra   s   @r   rc   rc   l   s    +("# " "r   rc   c                   `   ^  \ rS rSrS\R
                  4S jrSS.S jrS\4U 4S jjr	Sr
U =r$ )	_DoraConvNdLayer   r   c                     XU-  -   n[        [        SUR                  5       5      5      nUR                  SUSS9R	                  SS5      nU$ )Nr      T)pr   keepdimr   )tupleranger   r   r   )r   r   r    r!   r   r"   s         r   r#    _DoraConvNdLayer.get_weight_norm   sM    K//E!VZZ\*+kkA3k=GG1Mr   Nr@   c          
         UR                   nUR                   R                  S   n[        R                  " UR                   R	                  SU/5      UR                   R	                  US/5      5      n	U	R                  UR                  5      n	U R                   n
U R                  XyR                  5       U5      nUR                  5       nX-  nUc>  U R                  UUSUR                  UR                  UR                  UR                  S9nO9UR                  nUb*  SSUR                  5       S-
  -  -   nXmR                  " U6 -
  nUS-
  U-  X" U" U5      5      -  U-  -   nU$ )	rC   r   r)   N)rG   stridepaddingdilationgroups)r   r)   )r   ro   r   )r   r2   r   r3   r4   r5   r#   rF   conv_fnrv   rw   rx   ry   rG   r   )r   rJ   r:   r;   r!   r9   rA   r   r=   r    rL   r"   rM   rG   
bias_shaperO   s                   r   rP   _DoraConvNdLayer.forward   s_   
 ""MM"hhv}}112q':FMM<N<NPQSUw<WX!))&,,7KK	**63E3E3GQ "((*"0,,!(("**#,,!(( ' K ??D$t{/@1/D'EE
)IIz,BB%)[8>FSYZ[S\L];]`g;ggr   c                 *   > [         TU ]  5       nSU-   $ rS   rT   rV   s     r   rU   _DoraConvNdLayer.__repr__   rY   r   rj   )r/   rZ   r[   r\   r   r]   r#   rP   r^   rU   r_   r`   ra   s   @r   rl   rl      s1    u||  NR &P"# " "r   rl   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )DoraConv1dLayer   c                 N   > [         TU ]  U5        [        R                  U l        g r   )r   r   rH   conv1drz   r   s     r   r   DoraConv1dLayer.__init__       (xxr   rz   r/   rZ   r[   r\   r   r_   r`   ra   s   @r   r   r             r   r   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )DoraConv2dLayer   c                 N   > [         TU ]  U5        [        R                  U l        g r   )r   r   rH   conv2drz   r   s     r   r   DoraConv2dLayer.__init__   r   r   r   r   ra   s   @r   r   r      r   r   r   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )DoraConv3dLayer   c                 N   > [         TU ]  U5        [        R                  U l        g r   )r   r   rH   conv3drz   r   s     r   r   DoraConv3dLayer.__init__   r   r   r   r   ra   s   @r   r   r      r   r   r   )copyr   r   torch.nn.functionalr   
functionalrH   peft.utils.integrationsr   r   peft.utils.otherr   Moduler	   rc   rl   r   r   r   rj   r   r   <module>r      so         O &P"bii P"f" "43" 3"l &   &   &  r   