
    oiL                    *   S SK Jr  S SKrS SKJr  S SKrS SKrS SKJ	r	J
r
  S SKJr  S SKJr  S SKJr  SS	KJrJr  \
" 5       (       a    " S
 S\R(                  R*                  \5      r\	" 5       (       a!   " S S\R(                  R*                  \5      rgg)    )annotationsN)Optional)is_bnb_4bit_availableis_bnb_available)check_adapters_to_merge)dequantize_bnb_weight)	transpose   )RandLoraLayerUniqueBaseGradc                     ^  \ rS rSr     S
               SU 4S jjjrSSS jjrSS jrSSS jjrSS jrSS jr	SU 4S jjr
S	rU =r$ )Linear8bitLt    c
           
        > [         TU ]  5         [        R                  " X5        Xl        X l        U R                  UUUUUUU	S9  g N)randlora_alpharandlora_dropoutinit_weightssuper__init__r   fan_in_fan_out_active_adapterupdate_layerself
base_layeradapter_name
randlora_A
randlora_Brr   r   r   r   kwargs	__class__s              R/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/randlora/bnb.pyr   Linear8bitLt.__init__!   sR     G""44"0#/ -!1)      c                   [        X5      nU(       d  gU GH  nX0R                  R                  5       ;  a  M#  [        R                  " S5        U R                  U5      nU R                  5       R                  nU R                  5       R                  nUR                  c  UR                  Ul	        [        XV5      nUR                  UR                  5      R                  UR                  5      U-   nU(       a8  [        R                  " U5      R!                  5       (       d  [#        SU S35      e[$        R&                  R)                  UR                  S5      SUR*                  S9R                  UR                  5      U R                  5       l        UR-                  5         U R.                  R1                  U5        GM     g)  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged.
        Defaults to `None`.
Nz[Merge RandLora module to 8-bit linear may get different generations due to rounding errors.1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights)r   randlora_lambdakeyswarningswarnget_delta_weightget_base_layerweightstateSCBr   todtypedevicetorchisfiniteall
ValueErrorbnbnn
Int8Paramsr.   reset_gradsmerged_adaptersappend)	r   
safe_mergeadapter_namesactive_adapterrandlora_datar5   r6   outputw_datas	            r$   mergeLinear8bitLt.merge=   so    4DHM "/!)=)=)B)B)DDq !% 5 5n E,,.55++-3399$ &

EI.v==#6#67::=;O;OPS``ennV&<&@&@&B&B$KNK[[no  03vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#$$++N;5 #0r&   c                   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  Ga  U R                  R                  5       nXR                  R                  5       ;  a  MS  [        R                  " S5        U R                  U5      nU R                  5       R                  nU R                  5       R                  nUR                  c  UR                  Ul        [        X4S9nUR                  UR                  5      R                  UR                   5      U-
  n["        R$                  R'                  UR                  S5      SUR(                  S9R                  UR                   5      U R                  5       l
        UR+                  5         [        U R                  5      S:  a  GM  gg)	G
This method unmerges all merged adapter layers from the base weights.
Already unmerged. Nothing to doNr   z]Unmerge randlora module to 8-bit linear may get different generations due to rounding errors.)r6   r+   Fr,   )mergedr1   r2   lenrC   popr/   r0   r3   r4   r5   r6   r7   r   r8   r9   r:   r?   r@   rA   r.   rB   )r   rG   rH   r5   r6   rI   rJ   s          r$   unmergeLinear8bitLt.unmergek   s`    ;;?@d**+a/!%!5!5!9!9!;!)=)=)B)B)DDs !% 5 5n E,,.55++-3399$ &

EI.vC=#6#67::=;O;OPS``/2vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#) d**+a//r&   c                   U R                   U   nU R                  U   nUc  UR                  nUR                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                  U   R                  U5      nU R                  U   R                  U5      nU(       a@  UR                  5       nUR                  5       nUR                  5       nUR                  5       n[        U R                  U R                  5      [        U R                  U R                  5      pUSS2SU R                   2SU	24   R                  U5      nUSU
2SU R                   2SS24   R                  U5      nUR#                  SS9n[$        R&                  " XU5      R#                  SS9nXR                  :X  a  X4$ UR(                  UR(                  4$ a  
Performs scaling on the smallest random base (randlora_A) and returns randlora_A and randlora_B in the
correct order to fit the target layers' dimensions

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
Nr+   r
   )	start_dim)end_dimr   r    r:   r9   typer;   float16bfloat16r/   r8   randlora_gammafloatminout_featuresin_featuresmax	num_basesflattenr   applyTr   adapterr:   r   r    r9   cast_to_fp32r/   r]   min_dimmax_dimsliced_Asliced_Bupdate_Bupdate_As                  r$   get_scaled_basesLinear8bitLt.get_scaled_bases   s    1J1J~#**$$E
 ";;%/gUemm5K5fuX]XfXfOfL"227;>>vFO!009<<VDN'--/
'--/
"1"7"7"9!/!5!5!7  #4#4#4d6F6FGTM^M^`d`p`pIqW
 "!%5t~~%5xx"?@CCFKH!(7(,<dnn,<a"?@CCFKH  ''!'4H%++H~V^^gh^iH***))::xzz))r&   c                    U R                  U5      u  p#X#-  n[        X@R                  5      nU R                  U   nXV-  $ z
Compute the delta weight for the given adapter.

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
rp   r	   r   scalingr   rh   rn   ro   updateoutput_tensorru   s          r$   r3   Linear8bitLt.get_delta_weight   sF     "&!6!6w!?H(F%f.A.ABMll7+G **r&   c                   U R                   (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nGOU R                  (       a  U R                  " U/UQ70 UD6nGOnU R                  " U/UQ70 UD6nU R                   GHG  nXPR
                  R                  5       ;  a  M#  U R                  XQR                  S9u  pg[        R                  " 5       (       + nU(       a9  UR                  n	UR                  n
UR                  U
:w  a  UR                  U
5      nU R                  U   nU" UR                  UR                  5      5      n[        R                  R                  R!                  [        R                  R                  R!                  X5      U5      nU(       a  UR                  W	5      nU R"                  U   nXMU-  -   nGMJ     UR                  UR                  5      $ )a  
Perform the forward pass using the RandLora adapter.

Args:
    x (torch.Tensor): Input tensor.

Returns:
    torch.Tensor: Output tensor after applying the RandLora adaptation.

Note:
    This method implements the RandLora-specific forward pass. It applies the shared projections
    (randlora_A and randlora_B) along with the per-layer trainable parameters (lambda and gamma) to compute
    the adapter output.
r:   )disable_adaptersrP   rS   r   active_adaptersr/   r0   rp   r:   r;   is_autocast_enabledr9   r8   r   r@   
functionallinearru   r   xargsr"   resultrG   rn   ro   requires_conversionexpected_dtypecompute_dtypedropoutx_tempadapter_outputru   s                  r$   forwardLinear8bitLt.forward   s    $$;;LLN<T<V<<T<V<<T<V<&*&:&:N%-A-A-F-F-HH )-)>)>~V^V^)>)_&H.3.G.G.I*I'*)/(077m3 !] 3A"33NCG$QTT(..%9:F%*XX%8%8%?%?@S@S@Z@Z[a@lnv%wN*)7):):>)J"ll>:G#w&>>F- ';2 99QWW%%r&   c                *   > [         TU ]  5       nSU-   $ Nz	randlora.r   __repr__r   repr#   s     r$   r   Linear8bitLt.__repr__       '"$C$$r&   r   r   r   r   g        FTr   ztorch.nn.Moduler   strr!   intr   r   r   r^   r   boolr   r   returnNoneFNrE   r   rF   zOptional[list[str]]r   r   r   r   Nr   z list[torch.Tensor, torch.Tensor]r   torch.Tensorr   r   r   r   r   r   __name__
__module____qualname____firstlineno__r   rK   rS   rp   r3   r   r   __static_attributes____classcell__r#   s   @r$   r   r       s     "#&)#(!%	'	 	 	  	 $	 !	 	 	 	8,	<\	$<.	*`	+$0	&d	% 	%r&   r   c                     ^  \ rS rSr     S
               SU 4S jjjrSSS jjrSS jrSSS jjrSS jrSS jr	SU 4S jjr
S	rU =r$ )
Linear4biti  c
           
        > [         TU ]  5         [        R                  " X5        Xl        X l        U R                  UUUUUUU	S9  g r   r   r   s              r$   r   Linear4bit.__init__  sR     G""44"0#/ -!1)  r&   c                   [        X5      nU(       d  gU GH_  nX0R                  R                  5       ;  a  M#  [        R                  " S5        U R                  U5      nU R                  5       R                  nUR                  n[        R                  R                  UR                  UR                  5      U-   nU(       a8  [        R                  " U5      R!                  5       (       d  [#        SU S35      e[        R$                  R&                  " UR)                  S5      4SS0UD6R)                  UR*                  5      U R                  5       l        U R,                  R/                  U5        GMb     g)r(   Nz[Merge RandLora module to 4-bit linear may get different generations due to rounding errors.r)   r*   r+   r-   F)r   r/   r0   r1   r2   r3   r4   r5   __dict__r?   r   dequantize_4bitdataquant_stater;   r<   r=   r>   r@   
Params4bitr8   r:   rC   rD   )r   rE   rF   rG   rH   r5   r"   rJ   s           r$   rK   Linear4bit.merge   s.    4DHM "/!)=)=)B)B)DDq !% 5 5n E,,.5577VEWEWX[hhennV&<&@&@&B&B$KNK[[no  03vv/@/@5AQ/qaf/qjp/q/t/tMM0##%, $$++N;+ #0r&   c                   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  Ga8  U R                  R                  5       nXR                  R                  5       ;  a  MS  [        R                  " S5        U R                  U5      nU R                  5       R                  nUR                  n[        R                  R                  UR                  UR                   5      U-
  n[        R"                  R$                  " UR'                  S5      4SS0UD6R'                  UR(                  5      U R                  5       l
        [        U R                  5      S:  a  GM7  gg)rN   rO   Nr   z]Unmerge RandLora module to 4-bit linear may get different generations due to rounding errors.r+   r-   F)rP   r1   r2   rQ   rC   rR   r/   r0   r3   r4   r5   r   r?   r   r   r   r   r@   r   r8   r:   )r   rG   rH   r5   r"   rJ   s         r$   rS   Linear4bit.unmergeI  s    ;;?@d**+a/!%!5!5!9!9!;!)=)=)B)B)DDs !% 5 5n E,,.5577VEWEWX[hh/2vv/@/@5AQ/qaf/qjp/q/t/tMM0##%, d**+a//r&   c                   U R                   U   nU R                  U   nUc  UR                  nUR                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                  U   R                  U5      nU R                  U   R                  U5      nU(       a@  UR                  5       nUR                  5       nUR                  5       nUR                  5       n[        U R                  U R                  5      [        U R                  U R                  5      pUSS2SU R                   2SU	24   R                  U5      nUSU
2SU R                   2SS24   R                  U5      nUR#                  SS9n[$        R&                  " XU5      R#                  SS9nXR                  :X  a  X4$ UR(                  UR(                  4$ rV   rY   rg   s                  r$   rp   Linear4bit.get_scaled_basesb  s    1J1J~#**$$E
 ";;%/gUemm5K5fuX]XfXfOfL"227;>>vFO!009<<VDN'--/
'--/
"1"7"7"9!/!5!5!7  #4#4#4d6F6FGTM^M^`d`p`pIqW
 "!%5t~~%5xx"?@CCFKH!(7(,<dnn,<a"?@CCFKH''!'4H%++H~V^^gh^iH***))::xzz))r&   c                    U R                  U5      u  p#X#-  n[        X@R                  5      nU R                  U   nXV-  $ rs   rt   rv   s          r$   r3   Linear4bit.get_delta_weight  sF     "&!6!6w!?H(F%f.A.ABMll7+G **r&   c                   U R                   (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nGOU R                  (       a  U R                  " U/UQ70 UD6nGO~U R                  " U/UQ70 UD6nUR	                  5       nU R
                   GHG  nXPR                  R                  5       ;  a  M#  U R                  XQR                  S9u  pg[        R                  " 5       (       + nU(       a9  UR                  n	UR                  n
UR                  U
:w  a  UR                  U
5      nU R                  U   nU" UR                  UR                  5      5      n[        R                  R                   R#                  [        R                  R                   R#                  X5      U5      nU(       a  UR                  W	5      nU R$                  U   nXMU-  -   nGMJ     UR                  UR                  5      $ )Nr{   )r|   rP   rS   r   cloner}   r/   r0   rp   r:   r;   r~   r9   r8   r   r@   r   r   ru   r   s                  r$   r   Linear4bit.forward  s   $$;;LLN<T<V<<T<V<<T<V<&*&:&:N%-A-A-F-F-HH )-)>)>~V^V^)>)_&H.3.G.G.I*I'*)/(077m3 !] 3A"33NCG$QTT(..%9:F%*XX%8%8%?%?@S@S@Z@Z[a@lnv%wN*)7):):>)J"ll>:G#w&>>F- ';2 99QWW%%r&   c                *   > [         TU ]  5       nSU-   $ r   r   r   s     r$   r   Linear4bit.__repr__  r   r&   r   r   r   r   r   r   r   r   r   r   r   r   r   s   @r$   r   r     s     "#&)#(!%	'	 	 	  	 $	 !	 	 	 	6'	<R	2,	*\	+"#	&J	% 	%r&   r   )
__future__r   r1   typingr   bitsandbytesr?   r;   peft.import_utilsr   r   peft.tuners.tuners_utilsr   peft.utils.integrationsr   peft.utils.otherr	   layerr   r   r@   Moduler   r    r&   r$   <module>r      sq    #     E < 9 & 0 _%uxx _%D D%UXX__m D% r&   