
    oiE                    J   S SK Jr  S SKrS SKJrJr  S SKrS SKrS SK	J
r
Jr  S SKJrJr  S SKJr  SSKJr  SS	KJrJrJr  \" 5       (       a%   " S
 S\R.                  R0                  \5      rSS jr\
" 5       (       a&   " S S\R.                  R0                  \5      rSS jrgg)    )annotationsN)AnyOptional)is_bnb_4bit_availableis_bnb_available)BaseTunerLayercheck_adapters_to_merge)dequantize_bnb_weight   )RoadVariant)	RoadLayer_apply_road_get_delta_weightc                  ~   ^  \ rS rSr   S           S	U 4S jjjrS
SS jjrSS jrSS jrSU 4S jjrSr	U =r
$ )Linear8bitLt    c                ~   > [         TU ]  5         [        R                  " X5        X l        U R	                  UUUUS9  g N)variant
group_sizeinit_weightssuper__init__r   _active_adapterupdate_layerself
base_layeradapter_namer   r   r   kwargs	__class__s          N/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/road/bnb.pyr   Linear8bitLt.__init__"   C     Gt0#/ %)	      c                   [        X5      nU(       d  gU GH  nX0R                  ;   d  M  [        R                  " S5        U R	                  5       R
                  nU R	                  5       R                  nUR                  c  UR                  Ul        [        XES9n[        U R                  U   U R                  U   U R                  U   R                  U R                  U   R                  5      n[        R                   " XvR#                  UR$                  5      5      nUR#                  UR$                  5      R#                  UR&                  5      R)                  5       nU(       a8  [        R*                  " U5      R-                  5       (       d  [/        SU S35      e[0        R2                  R5                  UR#                  S5      SUR6                  S9R#                  UR&                  5      U R	                  5       l        U R	                  5       R8                  bw  U R	                  5       R8                  n	U	R$                  n
U	R                  n[        R                   " X{R#                  UR$                  5      5      nUR#                  U
5      U	l        UR;                  5         U R<                  R?                  U5        GM     g)	  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged.
        Defaults to `None`.
NzWMerge road module to 8-bit linear may get different generations due to rounding errors.state1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights) r	   _available_adapterswarningswarnget_base_layerweightr*   SCBr
   r   r   r   
road_thetadata
road_alphatorchmatmultodtypedevice
contiguousisfiniteall
ValueErrorbnbnn
Int8Paramsr0   biasreset_gradsmerged_adaptersappend)r   
safe_mergeadapter_namesactive_adapterr5   r*   outputroad_Rw_datarF   
orig_dtype	bias_datanew_biass                r#   mergeLinear8bitLt.merge6   s     4DHM "/!%=%==MMq "00299F //177Eyy($*JJ	 36GF.^477<<7<<	F #\\&))FLL2IJF#YYv||477FQQSF!%..*@*D*D*F*F(OP^O__rs  47663D3D		%(PVPgPg 4E 4b' '')0 **,11=#22499%)ZZ
$(II	#(<<V\\8R#S$,KK
$;	%%'((//?S #0r&   c                
   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  Ga  U R                  R                  5       nXR                  ;   Gaz  [        R                  " S5        U R                  5       R                  nU R                  5       R                  nUR                  c  UR                  Ul
        [        X#S9n[        U R                  U   U R                  U   U R                  U   R                   U R"                  U   R                   5      n[$        R&                  R)                  UR+                  [$        R,                  5      5      R+                  UR.                  5      n[$        R0                  " XdR+                  UR.                  5      5      nUR+                  UR.                  5      R+                  UR2                  5      R5                  5       n[6        R8                  R;                  UR+                  S5      SUR<                  S9R+                  UR2                  5      U R                  5       l        U R                  5       R>                  b^  U R                  5       R>                  nUR.                  n	UR                   n
[$        R0                  " Xj5      nUR+                  U	5      Ul        URA                  5         [        U R                  5      S:  a  GM  gg)	G
This method unmerges all merged adapter layers from the base weights.
 Already unmerged. Nothing to do.Nr   zYUnmerge road module to 8-bit linear may get different generations due to rounding errors.r)   r-   Fr.   )!mergedr2   r3   lenrH   popr1   r4   r5   r*   r6   r
   r   r   r   r7   r8   r9   r:   linalginvr<   float32r=   r;   r>   r?   rC   rD   rE   r0   rF   rG   )r   rL   r5   r*   rM   rN   
inv_road_RrO   rF   rP   rQ   rR   s               r#   unmergeLinear8bitLt.unmerges   s1    ;;@Ad**+a/!%!5!5!9!9!;!%=%==MMs "00299F //177Eyy($*JJ	26GF.^477<<7<<	F "'!1!1&))EMM2J!K!N!Nv||!\J"\\*ii6MNF#YYv||477FQQSF36663D3D		%(PVPgPg 4E 4b' '')0 **,11=#22499%)ZZ
$(II	#(<<
#F$,KK
$;	%%'G d**+a//r&   c                   U R                   (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ U R                  " U/UQ70 UD6nU R                   H  nXPR
                  ;  a  M  [        R                  " 5       (       + nU(       a4  UR                  nU R                  X@R                  U   R                  5      n[        U R                  U   U R                  U   U R                  U   U R                  U   U5      nU(       d  M  UR                  W5      nM     U$ Ndisable_adaptersrX   r_   r   active_adaptersr1   r:   is_autocast_enabledr=   _cast_input_dtyper7   r   r   r   r9   r<   r   xargsr!   resultrL   requires_conversionexpected_dtypes           r#   forwardLinear8bitLt.forward   s3   $$;;LLN<T<V<4 M3 <T<V<0 M- <T<V<&*&:&:N%-E-EE .3.G.G.I*I'*)/!%!7!7P^@_@e@e!f(^4777F +*DD0% ';( Mr&   c                *   > [         TU ]  5       nSU-   $ )Nzroad.r   __repr__r   repr"   s     r#   rr   Linear8bitLt.__repr__   s    '"$CS= r&   r   road_1@   Tr   torch.nn.Moduler    strr   r   r   intr   boolreturnNoneFNrJ   r~   rK   zOptional[list[str]]r   r   r   r   )ri   torch.Tensorrj   r   r!   r   r   r   r   r|   __name__
__module____qualname____firstlineno__r   rS   r_   rn   rr   __static_attributes____classcell__r"   s   @r#   r   r       so     $, !%	'	 	 !		
 	 	 	 	(;	@z+	(Z	@	! 	!r&   r   c                   S n[        U [        5      (       a  U R                  5       nOU nUR                  SS5      nU(       a  [        U[        R
                  R                  5      (       ac  UR                  5       nUR                  U R                  R                  U R                  R                  U R                  S.5        [        X40 UD6nU$ )Nloaded_in_8bitF)r0   	thresholdindex)
isinstancer   r4   getrC   rD   r   copyupdater*   r0   r   r   )targetr    r!   
new_moduletarget_base_layerr   eightbit_kwargss          r#   dispatch_bnb_8bitr      s    
fn-- & 5 5 7 &$4e<j):CFF<O<OPP$kkmO""(.(E(E!'!7!7#\\ &fNoNJr&   c                  ~   ^  \ rS rSr   S           S	U 4S jjjrS
SS jjrSS jrSS jrSU 4S jjrSr	U =r
$ )
Linear4bit   c                ~   > [         TU ]  5         [        R                  " X5        X l        U R	                  UUUUS9  g r   r   r   s          r#   r   Linear4bit.__init__   r%   r&   c                   [        X5      nU(       d  gU GH  nX0R                  ;   d  M  [        R                  " S5        U R	                  5       R
                  nUR                  n[        XDR                  S9n[        U R                  U   U R                  U   U R                  U   R                  U R                  U   R                  5      n[        R                   " XvR#                  UR$                  5      5      nUR#                  UR$                  5      R#                  UR&                  5      nU(       a8  [        R(                  " U5      R+                  5       (       d  [-        SU S35      eSU;   a  SUS'   SUS'   UR/                  S	S5        UR1                  5        V	V
s0 s H  u  pU	R3                  S
5      (       a  M  X_M!     nn	n
[4        R6                  R8                  " UR#                  S5      40 UD6R#                  UR&                  5      U R	                  5       l        U R	                  5       R:                  bw  U R	                  5       R:                  nUR$                  nUR                  n[        R                   " X}R#                  UR$                  5      5      nUR#                  U5      Ul        U R<                  R?                  U5        GM     gs  sn
n	f )r(   NzVMerge oft module to 4-bit linear may get different generations due to rounding errors.r)   r+   r,   bnb_quantizedFr/   r8   _r-   ) r	   r1   r2   r3   r4   r5   __dict__r
   quant_stater   r   r   r7   r8   r9   r:   r;   r<   r=   r>   r@   rA   rB   rZ   items
startswithrC   rD   
Params4bitrF   rH   rI   )r   rJ   rK   rL   r5   r!   rM   rN   rO   kvrF   rP   rQ   rR   s                  r#   rS   Linear4bit.merge   s<    4DHM "/!%=%==MMp "00299F#__F26ASASTF.^477<<7<<	F #\\&))FLL2IJF#YYv||477FF!%..*@*D*D*F*F(OP^O__rs  '&027/.3F?+JJvt,/5||~W~tqQ\\RUEVdad~FW36663D3DVYYuEU3`Y_3`3c3cdjdqdq3rD'')0**,11=#22499%)ZZ
$(II	#(<<V\\8R#S$,KK
$;	((//?Q #0< Xs   'KKc                   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  Ga  U R                  R                  5       nXR                  ;   GaT  [        R                  " S5        U R                  5       R                  nUR                  n[        X"R                  S9n[        U R                  U   U R                  U   U R                  U   R                   U R"                  U   R                   5      n[$        R&                  R)                  UR+                  [$        R,                  5      5      R+                  UR.                  5      n[$        R0                  " XdR+                  UR.                  5      5      nUR+                  UR.                  5      R+                  UR2                  5      nSU;   a  SUS'   SUS'   UR                  S	S5        [4        R6                  R8                  " UR+                  S
5      40 UD6R+                  UR2                  5      U R                  5       l        U R                  5       R:                  b^  U R                  5       R:                  nUR.                  n	UR                   n
[$        R0                  " Xj5      nUR+                  U	5      Ul        [        U R                  5      S:  a  GM  gg)rV   rW   Nr   zXUnmerge oft module to 4-bit linear may get different generations due to rounding errors.r)   r   Fr/   r8   r-   )rX   r2   r3   rY   rH   rZ   r1   r4   r5   r   r
   r   r   r   r   r7   r8   r9   r:   r[   r\   r<   r]   r=   r;   r>   rC   rD   r   rF   )r   rL   r5   r!   rM   rN   r^   rO   rF   rP   rQ   rR   s               r#   r_   Linear4bit.unmerge/  s    ;;@Ad**+a/!%!5!5!9!9!;!%=%==MMr "00299F#__F26ASASTF.^477<<7<<	F "'!1!1&))EMM2J!K!N!Nv||!\J"\\*ii6MNF#YYv||477FF&&027/.3F?+JJvt,36663D3DVYYuEU3`Y_3`3c3cdjdqdq3rD'')0**,11=#22499%)ZZ
$(II	#(<<
#F$,KK
$;	C d**+a//r&   c                   U R                   (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ U R                  " U/UQ70 UD6nU R                   H  nXPR
                  ;  a  M  [        R                  " 5       (       + nU(       a4  UR                  nU R                  X@R                  U   R                  5      n[        U R                  U   U R                  U   U R                  U   U R                  U   U5      nU(       d  M  UR                  W5      nM     U$ rb   rc   rh   s           r#   rn   Linear4bit.forwardZ  s5   $$;;LLN<T<V<> M= <T<V<: M7 <T<V< '+&:&:N%-E-EE .3.G.G.I*I'*)/!%!7!7P^@_@e@e!f(^4777F +*DD0# ';& Mr&   c                *   > [         TU ]  5       nSU-   $ )Nzoft.rq   rs   s     r#   rr   Linear4bit.__repr__  s    '"$CC<r&   rv   rw   rz   r   r   r   )ri   r   r   r   r   r   r   s   @r#   r   r      so     $, !%	'	 	 !		
 	 	 	 	(:	@x)	<V#	J	  	 r&   r   c                   S n[        U [        5      (       a  U R                  5       nOU nUR                  SS5      nU(       a  [	        5       (       a  [        U[
        R                  R                  5      (       ac  UR                  5       nUR                  UR                  UR                  R                  UR                  R                  S.5        [        X40 UD6nU$ )Nloaded_in_4bitF)compute_dtypecompress_statistics
quant_type)r   r   r4   r   r   rC   rD   r   r   r   r   r5   r   r   )r   r    r!   r   r   r   fourbit_kwargss          r#   dispatch_bnb_4bitr     s    
fn-- & 5 5 7 &$4e<355*EVX[X^X^XiXi:j:j#[[]N!!%6%D%D+<+C+C+W+W"3":":"E"E $FKNKJr&   )r   r{   r    r|   )
__future__r   r2   typingr   r   bitsandbytesrC   r:   peft.import_utilsr   r   peft.tuners.tuners_utilsr   r	   peft.utils.integrationsr
   configr   layerr   r   r   rD   Moduler   r   r   r    r&   r#   <module>r      s    #      E L 9  < < b!uxx	 b!H. d UXX__i d LQ r&   