
    oi?                    &   S SK Jr  S SKrS SKJr  S SKrS SKrS SKJ	r	J
r
  S SKJr  S SKJr  S SKJr  SS	KJr  \
" 5       (       a    " S
 S\R&                  R(                  \5      r\	" 5       (       a!   " S S\R&                  R(                  \5      rgg)    )annotationsN)Optional)is_bnb_4bit_availableis_bnb_available)check_adapters_to_merge)dequantize_bnb_weight)	transpose   )	VeraLayerc                     ^  \ rS rSr     S	               S
U 4S jjjrSSS jjrSS jrSS jrSS jrSU 4S jjr	Sr
U =r$ )Linear8bitLt    c
           
        > [         TU ]  5         [        R                  " X5        Xpl        X l        U R                  UUUUUUU	S9  g N)vera_dropoutinit_weights	d_initialsuper__init__r   fan_in_fan_out_active_adapterupdate_layerself
base_layeradapter_namevera_Avera_Brr   r   r   r   kwargs	__class__s              N/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/vera/bnb.pyr   Linear8bitLt.__init__!   R     Gt0"0#/ ))#      c                P   U R                   (       aO  [        R                  " SSR                  U R                  5       SSR                  U R
                  5       S35        [        X5      nU(       d  g U GH  nX0R                  R                  5       ;  a  M#  [        R                  " S5        U R                  U5      nU R                  5       R                  nU R                  5       R                  nUR                  c  UR                  Ul        [        XV5      nUR                  UR                   5      R                  UR"                  5      U-   nU(       a8  [$        R&                  " U5      R)                  5       (       d  [+        SU S35      e[,        R.                  R1                  UR                  S5      S	UR2                  S
9R                  UR"                  5      U R                  5       l        UR5                  5         U R                  R7                  U5        GM     g )N'Already following adapters were merged ,#. You are now additionally merging .zWMerge vera module to 8-bit linear may get different generations due to rounding errors.1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights)mergedwarningswarnjoinmerged_adaptersactive_adaptersr   vera_lambda_dkeysget_delta_weightget_base_layerweightstateSCBr   todtypedevicetorchisfiniteall
ValueErrorbnbnn
Int8Paramsr1   reset_gradsappend)	r   
safe_mergeadapter_namesactive_adapter	vera_datar<   r=   outputw_datas	            r#   mergeLinear8bitLt.merge=   s   {{=chhtG[G[>\=] ^88;AUAU8V7WWXZ
 4DHM "/!););)@)@)BBm !11.A	,,.55++-3399$ &

EI.v=9??366y7G7GH9TennV&<&@&@&B&B$KNK[[no  03vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#$$++N;5 #0r&   c                   U R                   (       d  [        R                  " S5        g [        U R                  5      S:  Ga  U R                  R                  5       nXR                  R                  5       ;  a  MS  [        R                  " S5        U R                  U5      nU R                  5       R                  nU R                  5       R                  nUR                  c  UR                  Ul        [        X4S9nUR                  UR                  5      R                  UR                   5      U-
  n["        R$                  R'                  UR                  S5      SUR(                  S9R                  UR                   5      U R                  5       l
        UR+                  5         [        U R                  5      S:  a  GM  g g )NAlready unmerged. Nothing to dor   zYUnmerge vera module to 8-bit linear may get different generations due to rounding errors.)r=   r.   Fr/   )r2   r3   r4   lenr6   popr8   r9   r:   r;   r<   r=   r>   r   r?   r@   rA   rF   rG   rH   r1   rI   )r   rM   rN   r<   r=   rO   rP   s          r#   unmergeLinear8bitLt.unmerged   s[   ;;?@d**+a/!%!5!5!9!9!;!););)@)@)BBo !11.A	,,.55++-3399$ &

EI.vC9??366y7G7GH9T/2vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#) d**+a//r&   c                $   U R                   U   nU R                  U   nUR                  nUR                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                  U   nU R                  U   nU(       a@  UR                  5       nUR                  5       nUR                  5       nUR                  5       nUSS2SU R                  24   R                  UR                  5      n	USU R                  2SS24   R                  UR                  5      n
UR                  S5      nUR                  S5      n[        X-  Xy-  -  U R                   5      nU(       a  UR                  US9nU$ )a  
Compute the delta weight for the given adapter.

Args:
    adapter (str): The name of the adapter for which the delta weight should be computed.

Returns:
    torch.Tensor: The computed delta weight for the VeRA adapter.

Note:
    This method implements the VeRA-specific weight update. Unlike LoRA, VeRA uses shared projection
    matrices (vera_A and vera_B) across all layers, along with per-layer trainable parameters (lambda_d and
    lambda_b).
r.   Nr@   r   r   rA   r@   typerB   float16bfloat16r8   vera_lambda_bfloatin_featuresr?   out_features	unsqueezer	   r   r   adapterr   r   rA   r@   cast_to_fp32lambda_dlambda_bsliced_Asliced_Boutput_tensors               r#   r:   Linear8bitLt.get_delta_weight   sd     [[)F[[)F ]]FLLE
 ";;%/gUemm5K5fuX]XfXfOfL))'2H))'2H#>>+#>>+a!34#3#3!33477HH1 1 1114588IH))"-H))"-H &x':x?R&SUYUhUhiM - 0 0u 0 =  r&   c           	        U R                   (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nGO
U R                  (       a  U R                  " U/UQ70 UD6nGOU R                  " U/UQ70 UD6nU R                   GH  nXPR
                  R                  5       ;  a  M#  U R
                  U   nU R                  U   nU R                  U   nU R                  U   n	U R                  U   n
[        R                  " 5       (       + nU(       a9  UR                  nUR                  nUR                  U:w  a  UR                  U5      nUSS2SU R                  24   R                  UR                   5      nU	SU R"                  2SS24   R                  UR                   5      nU
" UR                  UR                  5      5      nU[        R$                  R&                  R)                  U[        R$                  R&                  R)                  UU5      -  U5      -  nU(       a  UR                  W5      nUU-   nGM     UR                  UR                  5      $ )a  
Perform the forward pass using the VeRA adapter.

Args:
    x (torch.Tensor): Input tensor.

Returns:
    torch.Tensor: Output tensor after applying the VeRA adaptation.

Note:
    This method implements the VeRA-specific forward pass. It applies the shared projections (vera_A and
    vera_B) along with the per-layer trainable parameters (lambda_d and lambda_b) to compute the adapter
    output.
N)disable_adaptersr2   rW   r   r7   r8   r9   r`   r   r   r   rB   is_autocast_enabledr@   r?   rb   rA   rc   rG   
functionallinearr   xargsr!   resultrM   rh   ri   r   r   dropoutrequires_conversionexpected_dtypecompute_dtyperj   rk   x_tempadapter_outputs                     r#   forwardLinear8bitLt.forward   s    $$;;LLN<T<V<<T<V<<T<V<&*&:&:N%-?-?-D-D-FF #11.AH#11.AH![[8F![[8F"//?G.3.G.G.I*I'*)/(077m3 !] 3A%a);4+;+;);&;<??IH%&9(9(9&91&<=@@JH$QTT(..%9:F%-0C0C0J0J 588#6#6#=#=fh#OOQY1 &N +)7):):>)J#n4F? ';D 99QWW%%r&   c                *   > [         TU ]  5       nSU-   $ Nzvera.r   __repr__r   repr"   s     r#   r   Linear8bitLt.__repr__       '"$CS= r&   r   r   r   g        FTg?r   ztorch.nn.Moduler   strr    intr   ra   r   boolr   r   r   ra   returnNoneFNrK   r   rL   zOptional[list[str]]r   r   r   r   r   torch.Tensorrt   r   r   r   r   r   __name__
__module____qualname____firstlineno__r   rQ   rW   r:   r}   r   __static_attributes____classcell__r"   s   @r#   r   r       s     "%#(!%"	'	 	 	  	 !	 	 	 	 	8%	<N	$63	!j9	&v	! 	!r&   r   c                     ^  \ rS rSr     S	               S
U 4S jjjrSSS jjrSS jrSS jrSS jrSU 4S jjr	Sr
U =r$ )
Linear4bit   c
           
        > [         TU ]  5         [        R                  " X5        Xpl        X l        U R                  UUUUUUU	S9  g r   r   r   s              r#   r   Linear4bit.__init__   r%   r&   c                :   U R                   (       aO  [        R                  " SSR                  U R                  5       SSR                  U R
                  5       S35        [        X5      nU(       d  g U GH  nX0R                  R                  5       ;  a  M#  [        R                  " S5        U R                  U5      nU R                  5       R                  nUR                  nUR                  5        VVs0 s H  u  pxUR                  S5      (       a  M  Xx_M!     nnn[        R                   R#                  UR$                  UR&                  5      U-   n	U(       a8  [(        R*                  " U	5      R-                  5       (       d  [/        SU S35      e[        R0                  R2                  " U	R5                  S	5      4S
S0UD6R5                  UR6                  5      U R                  5       l        U R                  R9                  U5        GM     g s  snnf )Nr(   r)   r*   r+   zWMerge vera module to 4-bit linear may get different generations due to rounding errors._r,   r-   r.   r0   F)r2   r3   r4   r5   r6   r7   r   r8   r9   r:   r;   r<   __dict__items
startswithrF   rq   dequantize_4bitdataquant_staterB   rC   rD   rE   rG   
Params4bitr?   rA   rJ   )
r   rK   rL   rM   rN   r<   r!   kvrP   s
             r#   rQ   Linear4bit.merge  s   {{=chhtG[G[>\=] ^88;AUAU8V7WWXZ
 4DHM "/!););)@)@)BBm !11.A	,,.55+1<<>S>41cAR$!$>S77VEWEWX[ddennV&<&@&@&B&B$KNK[[no  03vv/@/@5AQ/qaf/qjp/q/t/tMM0##%, $$++N;/ #0 Ts   9HHc                   U R                   (       d  [        R                  " S5        g [        U R                  5      S:  Ga8  U R                  R                  5       nXR                  R                  5       ;  a  MS  [        R                  " S5        U R                  U5      nU R                  5       R                  nUR                  n[        R                  R                  UR                  UR                   5      U-
  n[        R"                  R$                  " UR'                  S5      4SS0UD6R'                  UR(                  5      U R                  5       l
        [        U R                  5      S:  a  GM7  g g )NrT   r   zYUnmerge vera module to 4-bit linear may get different generations due to rounding errors.r.   r0   F)r2   r3   r4   rU   r6   rV   r8   r9   r:   r;   r<   r   rF   rq   r   r   r   rG   r   r?   rA   )r   rM   rN   r<   r!   rP   s         r#   rW   Linear4bit.unmerge7  s   ;;?@d**+a/!%!5!5!9!9!;!););)@)@)BBo !11.A	,,.5577VEWEWX[dd/2vv/@/@5AQ/qaf/qjp/q/t/tMM0##%, d**+a//r&   c                $   U R                   U   nU R                  U   nUR                  nUR                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                  U   nU R                  U   nU(       a@  UR                  5       nUR                  5       nUR                  5       nUR                  5       nUS S 2S U R                  24   R                  UR                  5      n	US U R                  2S S 24   R                  UR                  5      n
UR                  S5      nUR                  S5      n[        X-  Xy-  -  U R                   5      nU(       a  UR                  US9nU$ )Nr.   rZ   r[   r\   re   s               r#   r:   Linear4bit.get_delta_weightM  s\   [[)F[[)F]]FLLE!;;%/gUemm5K5fuX]XfXfOfL))'2H))'2H#>>+#>>+a!34#3#3!33477HH1 1 1114588IH))"-H))"-H%x':x?R&SUYUhUhiM - 0 0u 0 =  r&   c           	         U R                   (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nGOU R                  (       a  U R                  " U/UQ70 UD6nGOU R                  " U/UQ70 UD6nUR	                  5       nU R
                   GH  nXPR                  R                  5       ;  a  M#  U R                  U   nU R                  U   nU R                  U   nU R                  U   n	U R                  U   n
[        R                  " 5       (       + nU(       a9  UR                  nUR                  nUR                  U:w  a  UR                  U5      nUS S 2S U R                   24   R                  UR"                  5      nU	S U R$                  2S S 24   R                  UR"                  5      nU
" UR                  UR                  5      5      nU[        R&                  R(                  R+                  U[        R&                  R(                  R+                  UU5      -  U5      -  nU(       a  UR                  W5      nUU-   nGM     UR                  UR                  5      $ )N)ro   r2   rW   r   cloner7   r8   r9   r`   r   r   r   rB   rp   r@   r?   rb   rA   rc   rG   rq   rr   rs   s                     r#   r}   Linear4bit.forwardk  s   $$;;LLN<T<V<<T<V<<T<V<&*&:&:N%-?-?-D-D-FF #11.AH#11.AH![[8F![[8F"//?G.3.G.G.I*I'*)/(077m3 !] 3A%a);4+;+;);&;<??IH%&9(9(9&91&<=@@JH$QTT(..%9:F%-0C0C0J0J 588#6#6#=#=fh#OOQY1 &N +)7):):>)J#n4F? ';D 99QWW%%r&   c                *   > [         TU ]  5       nSU-   $ r   r   r   s     r#   r   Linear4bit.__repr__  r   r&   r   r   r   r   r   r   r   r   r   r   r   s   @r#   r   r      s     "%#(!%"	'	 	 	  	 !	 	 	 	 	8"	<H	,	!<,	&\	! 	!r&   r   )
__future__r   r3   typingr   bitsandbytesrF   rB   peft.import_utilsr   r   peft.tuners.tuners_utilsr   peft.utils.integrationsr   peft.utils.otherr	   layerr   rG   Moduler   r    r&   r#   <module>r      sq    #     E < 9 &  Q!uxx	 Q!h e!UXX__i e! r&   