
    oi	>                    2   S SK Jr  S SKrS SKJr  S SKrS SKrS SKJ	r	J
r
  S SKJrJr  S SKJr  SSKJr  \
" 5       (       a%   " S	 S
\R$                  R&                  \5      rSS jr\	" 5       (       a&   " S S\R$                  R&                  \5      rSS jrgg)    )annotationsN)Optional)is_bnb_4bit_availableis_bnb_available)BaseTunerLayercheck_adapters_to_merge)dequantize_bnb_weight   )OFTLayerc                     ^  \ rS rSr         S	                       S
U 4S jjjrSSS jjrSS jrS rSS jrSU 4S jjr	Sr
U =r$ )Linear8bitLt   c                   > [         TU ]  5         [        R                  " X5        SU l        X l        U R                  UUUUUUU	UU
US9
  g NF)oft_block_sizemodule_dropoutcoftepsblock_shareinit_weightsuse_cayley_neumannnum_cayley_neumann_termssuper__init__r   fan_in_fan_out_active_adapterupdate_layer)self
base_layeradapter_namerr   r   r   r   r   r   r   r   kwargs	__class__s                M/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/oft/bnb.pyr   Linear8bitLt.__init__!   ]     Gd/"'D#/ --')#5)A      c                D   [        X5      nU(       d  gU GH  nX0R                  R                  5       ;  a  M#  [        R                  " S5        U R                  5       R                  nU R                  5       R                  nUR                  c  UR                  Ul        [        XES9nU R                  U5      n[        R                  " USS5      n[        R                  " XvR                  UR                  5      5      n[        R                  " USS5      nUR                  UR                  5      R                  UR                   5      nU(       a8  [        R"                  " U5      R%                  5       (       d  ['        SU S35      e[(        R*                  R-                  UR                  S5      S	UR.                  S
9R                  UR                   5      U R                  5       l        UR1                  5         U R2                  R5                  U5        GM	     g)  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged.
        Defaults to `None`.
NzVMerge oft module to 8-bit linear may get different generations due to rounding errors.stater   r
   1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights)r   oft_Rkeyswarningswarnget_base_layerweightr,   SCBr	   get_delta_weighttorch	transposemmtodtypedeviceisfiniteall
ValueErrorbnbnn
Int8Paramsr2   reset_gradsmerged_adaptersappend)	r   
safe_mergeadapter_namesactive_adapterr8   r,   outputoft_dataw_datas	            r%   mergeLinear8bitLt.mergeB   s    4DHM "/!)::vw,,.55++-3399$ &

EI /vC00@A6(IIhnn,EFA68>>255hooFennV&<&@&@&B&B$KNK[[no  03vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#$$++N;? #0r(   c                d   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  Ga  U R                  R                  5       nXR                  R                  5       ;  a  MS  [        R                  " S5        U R                  5       R                  nU R                  5       R                  nUR                  c  UR                  Ul        [        X#S9nU R                  U5      n[        R                  " USS5      n[        R                   " UR#                  5       UR%                  UR&                  5      5      n[        R                  " USS5      nUR%                  UR&                  5      R%                  UR(                  5      n[*        R,                  R/                  UR%                  S5      SUR0                  S	9R%                  UR(                  5      U R                  5       l	        UR3                  5         [        U R                  5      S:  a  GM  gg)
G
This method unmerges all merged adapter layers from the base weights.
 Already unmerged. Nothing to do.Nr   zXUnmerge oft module to 8-bit linear may get different generations due to rounding errors.r+   r
   r/   Fr0   )mergedr5   r6   lenrH   popr3   r4   r7   r8   r,   r9   r	   r:   r;   r<   r=   tr>   r?   r@   rD   rE   rF   r2   rG   )r   rL   r8   r,   rM   rN   rO   s          r%   unmergeLinear8bitLt.unmergeu   s    ;;@Ad**+a/!%!5!5!9!9!;!)::n ,,.55++-3399$ &

EI.vC00@A6(**,		(..0IJA68>>255hooF/2vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#3 d**+a//r(   c                <    U R                   U   R                  5       $ Nr3   
get_weightr   adapters     r%   r:   Linear8bitLt.get_delta_weight       ::g&1133r(   c                   U R                   (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ U R                   H  nXPR
                  R                  5       ;  a  M"  U R
                  U   n[        R                  " 5       (       + nU(       a1  UR                  nU R                  XR                  R                  5      nU" U5      nU(       d  M  UR                  W5      nM     U R                  " U/UQ70 UD6nU$ r\   disable_adaptersrU   rY   r    active_adaptersr3   r4   r;   is_autocast_enabledr?   _cast_input_dtyper8   r>   	r   xargsr#   resultrL   r3   requires_conversionexpected_dtypes	            r%   forwardLinear8bitLt.forward   s
   $$;;LLN<T<V<( M' <T<V<$ M! '+&:&:N%ZZ__->>  JJ~6E.3.G.G.I*I'*)* 221ll6H6HIaA**DD0 '; <T<V<Mr(   c                *   > [         TU ]  5       nSU-   $ Nzoft.r   __repr__r   repr$   s     r%   rt   Linear8bitLt.__repr__       '"$CC<r(   r   r   )	   r           TFiUMu?FF   )r    torch.nn.Moduler!   strr"   intr   r   r   floatr   boolr   r   r   r   r   r   r   r   r   r   returnNoneFNrJ   r   rK   zOptional[list[str]]r   r   r   r   rj   torch.Tensorr   r   r   r   __name__
__module____qualname____firstlineno__r   rP   rY   r:   ro   rt   __static_attributes____classcell__r$   s   @r%   r   r      s     "#$'!% %',,-	'	 	 		
  	 "	 	 	 	 	 !%	 '*	 	 	B1	<f!	$F	4	4	  	 r(   r   c                   S n[        U [        5      (       a  U R                  5       nOU nUR                  SS5      nU(       a  [        U[        R
                  R                  5      (       ac  UR                  5       nUR                  U R                  R                  U R                  R                  U R                  S.5        [        X40 UD6nU$ )Nloaded_in_8bitF)r2   	thresholdindex)
isinstancer   r7   getrD   rE   r   copyupdater,   r2   r   r   )targetr!   r#   
new_moduletarget_base_layerr   eightbit_kwargss          r%   dispatch_bnb_8bitr      s    
fn-- & 5 5 7 &$4e<j):CFF<O<OPP$kkmO""(.(E(E!'!7!7#\\ &fNoNJr(   c                     ^  \ rS rSr         S	                       S
U 4S jjjrSSS jjrSS jrS rSS jrSU 4S jjr	Sr
U =r$ )
Linear4bit   c                   > [         TU ]  5         [        R                  " X5        SU l        X l        U R                  UUUUUUUU	U
US9
  g r   r   )r   r    r!   r"   r   r   r   r   r   r   r   r   r#   r$   s                r%   r   Linear4bit.__init__   r'   r(   c                   [        X5      nU(       d  gU GH*  nX0R                  R                  5       ;  a  M#  [        R                  " S5        U R                  5       R                  nUR                  n[        XDR                  S9nU R                  U5      n[        R                  " USS5      n[        R                  " XvR                  UR                  5      5      n[        R                  " USS5      nUR                  UR                  5      R                  UR                   5      nU(       a8  [        R"                  " U5      R%                  5       (       d  ['        SU S35      eSU;   a  S	US'   S	US
'   UR)                  SS5        UR+                  5        V	V
s0 s H  u  pU	R-                  S5      (       a  M  X_M!     nn	n
[.        R0                  R2                  " UR                  S5      40 UD6R                  UR                   5      U R                  5       l        U R4                  R7                  U5        GM-     gs  sn
n	f )r*   NzVMerge oft module to 4-bit linear may get different generations due to rounding errors.r+   r   r
   r-   r.   bnb_quantizedFr1   data_r/   )r   r3   r4   r5   r6   r7   r8   __dict__r	   quant_stater:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rW   items
startswithrD   rE   
Params4bitrH   rI   )r   rJ   rK   rL   r8   r#   rM   rN   rO   kvs              r%   rP   Linear4bit.merge   s    4DHM "/!)::vw,,.55.v=O=OP00@A6(IIhnn,EFA68>>255hooFennV&<&@&@&B&B$KNK[[no  #f,.3F?+*/'

64(+1<<>S>41cAR$!$>S/2vv/@/@5AQ/\U[/\/_/_`f`m`m/n##%,$$++N;= #06 Ts    I Ic                4   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  Ga  U R                  R                  5       nXR                  R                  5       ;  a  MS  [        R                  " S5        U R                  5       R                  nUR                  n[        X"R                  S9nU R                  U5      n[        R                  " USS5      n[        R                   " UR#                  5       UR%                  UR&                  5      5      n[        R                  " USS5      nUR%                  UR&                  5      R%                  UR(                  5      nSU;   a  SUS'   SUS	'   UR                  S
S5        [*        R,                  R.                  " UR%                  S5      40 UD6R%                  UR(                  5      U R                  5       l	        [        U R                  5      S:  a  GM  gg)rS   rT   Nr   zXUnmerge oft module to 4-bit linear may get different generations due to rounding errors.r+   r
   r   Fr1   r   r/   )rU   r5   r6   rV   rH   rW   r3   r4   r7   r8   r   r	   r   r:   r;   r<   r=   rX   r>   r?   r@   rD   rE   r   )r   rL   r8   r#   rM   rN   rO   s          r%   rY   Linear4bit.unmerge'  s    ;;@Ad**+a/!%!5!5!9!9!;!)::n ,,.55.v=O=OP00@A6(**,		(..0IJA68>>255hooF"f,.3F?+*/'

64(/2vv/@/@5AQ/\U[/\/_/_`f`m`m/n##%,/ d**+a//r(   c                <    U R                   U   R                  5       $ r\   r]   r_   s     r%   r:   Linear4bit.get_delta_weightH  rb   r(   c                   U R                   (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ U R                   H  nXPR
                  R                  5       ;  a  M"  U R
                  U   n[        R                  " 5       (       + nU(       a1  UR                  nU R                  XR                  R                  5      nU" U5      nU(       d  M  UR                  W5      nM     U R                  " U/UQ70 UD6nU$ r\   rd   ri   s	            r%   ro   Linear4bit.forwardK  s
   $$;;LLN<T<V<6 M5 <T<V<2 M! '+&:&:N%ZZ__->>  JJ~6E.3.G.G.I*I'*)* 221ll6H6HIaA**DD0 '; <T<V<Mr(   c                *   > [         TU ]  5       nSU-   $ rr   rs   ru   s     r%   rt   Linear4bit.__repr__l  rx   r(   ry   )	rz   r   r{   Fr|   FTFr}   )r    r~   r!   r   r"   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   s   @r%   r   r      s     "#$' %!%',,-	'	 	 		
  	 "	 	 	 	 	 !%	 '*	 	 	B0	<d	oB	4	B	  	 r(   r   c                   S n[        U [        5      (       a  U R                  5       nOU nUR                  SS5      nU(       a  [	        5       (       a  [        U[
        R                  R                  5      (       ac  UR                  5       nUR                  UR                  UR                  R                  UR                  R                  S.5        [        X40 UD6nU$ )Nloaded_in_4bitF)compute_dtypecompress_statistics
quant_type)r   r   r7   r   r   rD   rE   r   r   r   r   r8   r   r   )r   r!   r#   r   r   r   fourbit_kwargss          r%   dispatch_bnb_4bitr   p  s    
fn-- & 5 5 7 &$4e<355*EVX[X^X^XiXi:j:j#[[]N!!%6%D%D+<+C+C+W+W"3":":"E"E $FKNKJr(   )r   r~   r!   r   )
__future__r   r5   typingr   bitsandbytesrD   r;   peft.import_utilsr   r   peft.tuners.tuners_utilsr   r   peft.utils.integrationsr	   layerr   rE   Moduler   r   r   r    r(   r%   <module>r      sz    #     E L 9  X uxx X t. \ UXX__h \ |A r(   