
    oi&                        S SK Jr  S SKJrJrJr  S SKrS SKJr  S SKJ	r	  S SK
Jr  SSKJr   " S S	5      r " S
 S\5      r " S S\5      r " S S\5      rg)    )annotations)AnyCallableOptionalN)Tensor)lora   )XLoraConfigc                  \    \ rS rSrSr            SS jr \S	S j5       r S
S jrSr	g)
XLoraLayer   z
A XLoraLayer wraps any LoraLayer and performs the XLora operation on the LoRA adaptors specified. Its primary API
is the forward method, which uses the scalings to execute the XLora algorithm.
c                @    Xl         X0l        X l        X@l        XPl        g N)modeltarget_forwardtargetlayer_numberconfig)selfr   r   r   r   r   s         Q/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/xlora/layer.py__init__XLoraLayer.__init__!   s     
,(    c                @    US S 2S S 2U4   R                  S5      nX-  $ )N)	unsqueeze)xscalings_layeradapterscalingss       r   apply_scalings_to_xXLoraLayer.apply_scalings_to_x3   s'     "!Q-0::2>|r   c                *   US S 2S S 2U R                   S S 24   nU R                  R                  b~  [        R                  " X R                  R                  SS9u  p4[        R
                  " U[        R                  S9nUR                  SUS5        X%R                  UR                  5      -  nU R                  R                  (       aI  US:g  nUR                  U) [        S5      5      n[        R                  " USS9nUR                  U) S5      nU$ )	Nr   )kdim)dtypeTr   z-inf)r%   g        )r   r   
top_k_loratorchtopk
zeros_likeboolscatter_tor&   enable_softmax_topkmasked_fillfloatsoftmax)	r   r    xlora_scalings_topk_indicesmasknonzero_maskfullnew_scalingss	            r   get_maybe_topk_scalings"XLoraLayer.get_maybe_topk_scalings?   s    !)!Q0A0A1*D!E;;!!-#jj;;;Q;QWYZOA ##N%**EDMM"lD1+ggn6J6J.KKN ;;**)Q.L!--|mU6]KD ==26L)55|mSINr   )r   r   r   r   r   N)r   	nn.Moduler   zlora.LoraLayerr   Callable[..., Any]r   intr   r
   returnNone)r   torch.Tensorr   r@   r   r=   r>   r@   )r>   r@   )
__name__
__module____qualname____firstlineno____doc__r   staticmethodr!   r9   __static_attributes__ r   r   r   r      si    
  +	
   
  
r   r   c                  V   ^  \ rS rSr            SU 4S jjrSS.SS jjrSrU =r$ )	XLoraLinearLayerV   c                (   > [         TU ]  XX4U5        g r   superr   r   r   r   r   r   r   	__class__s         r   r   XLoraLinearLayer.__init__W        	fMr   Nr    c          	        UR                   nUb  U R                  U5      nU R                  R                  " U/UQ70 UD6nU R                  R                  (       GdR  [        U R                  R                  5       GH.  u  pXR                  R                  R                  5       ;  a  M/  U R                  R                  U	   (       a  [        S5      eU R                  R                  U	   n
U R                  R                  U	   nU R                  R                  U	   nU R                  R                  U	   nUR                  U
R                  R                   5      nUb*  U R!                  UWU5      nU R"                  R$                  nOUnSnX{" U
" U" U5      5      5      U-  U-  -  nGM1     UR                  U5      nU$ 
This method is designed to be a drop-in-replacement for the LoRA layers' .forward method. To use it, a bound
method must be created (bound to an instance of the XLoraLayer class).
7X-LoRA currently does not support LoRA layers with DoRAr	   r&   r9   r   
base_layermerged	enumerateactive_adapterslora_Akeysuse_dora
ValueErrorlora_Blora_dropoutscalingr-   weightr!   r   global_scaling_weightr   r   r    argskwargsprevious_dtyper2   result	adapter_nactive_adapterr]   ra   dropoutrc   x_modscaling_weights                   r   forwardXLoraLinearLayer.forwarda   s|    !99(CN'';D;F; {{!!!-6t{{7R7R-S)	!););)@)@)BB;;''7$%^__++N;++N;++22>B++--n=DD,,-' 44Q	RE%)[[%F%FNE%&N&!787B^SS# .T& >*r   rH   )r   r;   r   zlora.Linearr   r<   r   r=   r   r
   r>   r?   
r   r   rg   r   r    zOptional[Tensor]rh   r   r>   r   rA   rB   rC   rD   r   rp   rG   __classcell__rP   s   @r   rJ   rJ   V   sZ    NN N +	N
 N N 
N KO " "r   rJ   c                  V   ^  \ rS rSr            SU 4S jjrSS.SS jjrSrU =r$ )	XLoraEmbeddingLayer   c                (   > [         TU ]  XX4U5        g r   rM   rO   s         r   r   XLoraEmbeddingLayer.__init__   rR   r   NrS   c                  Ub  U R                  U5      nU R                  R                  " U/UQ70 UD6nU R                  R                  5       nU R                  R                  (       GdV  [        U R                  R                  5       GH2  u  pXR                  R                  ;  a  M!  U R                  R                  R                  U	S5      (       a  [        S5      eU R                  R                  U	   R                  n
U R                  R                  U	   R                  nU R                  R                  U	   nU R                  R                  X5      nUb*  U R                  UWU5      nU R                   R"                  nOUnSnX-  U-  U-  nUb  UUR%                  UR&                  5      -  nUU-  nGM5     U$ )rV   FrW   r	   )r9   r   rY   _get_embed_scalerZ   r[   r\   lora_embedding_Ar_   getr`   Tlora_embedding_Brc   _embedr!   r   re   r-   r&   )r   r   r    rg   rh   r2   rj   embed_scalerk   rl   embedding_Aembedding_Brc   after_Aafter_A_modro   adapter_outputs                    r   rp   XLoraEmbeddingLayer.forward   s    !99(CN'';D;F; kk224 {{!!!-6t{{7R7R-S)	!)E)EE;;''++NEBB$%^__"kk::>JLL"kk::>JLL++--n=++,,Q<'"&":":7NT]"^K%)[[%F%FN")K%&N"-";w!F!W *%3knn^EYEY6Z%ZN.(/ .T2 r   rH   )r   r;   r   zlora.Embeddingr   r<   r   r=   r   r
   r>   r?   rr   rs   ru   s   @r   rw   rw      sZ    NN N +	N
 N N 
N KO * *r   rw   c                  V   ^  \ rS rSr            SU 4S jjrSS.SS jjrSrU =r$ )	XLoraConv2dLayer   c                (   > [         TU ]  XX4U5        g r   rM   rO   s         r   r   XLoraConv2dLayer.__init__   rR   r   NrS   c          	        UR                   nUb  U R                  U5      nU R                  R                  " U/UQ70 UD6nU R                  R                  (       GdR  [        U R                  R                  5       GH.  u  pXR                  R                  R                  5       ;  a  M/  U R                  R                  U	   (       a  [        S5      eU R                  R                  U	   n
U R                  R                  U	   nU R                  R                  U	   nU R                  R                  U	   nUR                  U
R                  R                   5      nUb*  U R!                  UWU5      nU R"                  R$                  nOUnSnX{" U
" U" U5      5      5      U-  U-  -  nGM1     UR                  U5      nU$ rU   rX   rf   s                   r   rp   XLoraConv2dLayer.forward   s|    !99(CN'';D;F; {{!!!-6t{{7R7R-S)	!););)@)@)BB;;''7$%^__++N;++N;++22>B++--n=DD,,-' 44Q	RE%)[[%F%FNE%&N&!787B^SS# .T& >*r   rH   )r   r;   r   zlora.Conv2dr   r<   r   r=   r   r
   r>   r?   rr   rs   ru   s   @r   r   r      sZ    NN N +	N
 N N 
N KO # #r   r   )
__future__r   typingr   r   r   r(   torch.nnnnr   peft.tunersr   r   r
   r   rJ   rw   r   rH   r   r   <module>r      sP    # * *     8 8v-z -`5* 5p.z .r   