
    oi7                        S SK Jr  S SKrS SKrS SKrS SKJrJrJr  S SK	r	S SK
Jr  S SKJs  Jr  S SKJrJr  S SKJr  S SKJr  SSKJr   " S	 S
\R.                  \5      r        SS jrg)    )annotationsN)AnyOptionalUnion)BaseTunerLayercheck_adapters_to_merge)	transpose)gather_params_ctx   )	LoraLayerc                     ^  \ rS rSrSr         S                   SU 4S jjjrS\R                  SSS4 SS jjrSS jr	SSS jjr
SS	 jrSS
 jrSU 4S jjrSrU =r$ )LoraParallelLinear    a|  
When the target layer parallel_linear is RowParallelLinear, in order to keep the input and output shapes
consistent, we need to split the lora matrix A into rows, and the lora_B at this time should be a complete linear
layer; In the same way, when the target layer is ColumnParallelLinear, we perform column segmentation on lora_B,
while lora_A is still a complete linear layer.
FTc                  > U(       a"  [        U R                  R                   S35      e[        TU ]  5         [
        R                  " U 4SU0UD6  U(       a"  [        U R                  R                   S35      eX0l        [        XR                  5      U l	        Xpl
        X l        US   nSU0n[        R                  n[        US5      (       a  UR                  nSnSnU R                  (       a  UR                   nOUR"                  nU R$                  " UU4UUU	U
UUUUS.UD6  U(       a"  [        U R                  R                   S	35      eSU l        g )
Nz0 does not support lora_bias yet, set it to False
base_layerz2 does not support DoRA yet, please set it to Falsemegatron_configinit_methodTF)
lora_alphalora_dropoutinit_lora_weights
use_rslorause_dorar   input_is_parallelgather_outputzB does not support target_conv_1d_layer yet, please set it to False)
ValueError	__class____name__super__init__r   backend
isinstanceRowParallelLinearis_parallel_afan_in_fan_out_active_adapterinitxavier_normal_hasattrr   r   r   update_layeris_target_conv_1d_layer)selfr   adapter_namer    rr   r   r$   r*   r   r   r   	lora_biaskwargsr   parallel_linear_kwargsr   r   r   r   s                      S/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/lora/tp_layer.pyr   LoraParallelLinear.__init__(   sa      7 788hijj4AJA&A 7 788jkll'
4M4MN,+ !23"3_!E))?M22)55K  * < <&44M	
 "%/!#/'	
 %	
 #>>**++mn  (-$    c           
        [        5       R                  5       nUS	 US::  a  [        SU 35      eX R                  U'   X0R                  U'   US:  a  [
        R                  " US9nO[
        R                  " 5       nXR                  U'   US   n[        R                  Ul        U R                  (       aX  U R                  R                  U R                  USU	SUUS	9n[
        R                   " X R"                  S[        R                  S
9nOW[
        R                   " U R                  US[        R                  S
9nU R                  R%                  UU R"                  SU
UUS9nUU R&                  U'   UU R(                  U'   U(       a'  U[*        R,                  " U5      -  U R.                  U'   OX2-  U R.                  U'   XpR0                  U'   [3        U[4        5      (       aU  UR7                  S5      (       a?  [9        U R;                  5       R<                  5         U R?                  X5        S S S 5        GO,[3        U[4        5      (       aT  UR7                  S5      (       a>  [9        U R;                  5       R<                  5         U RA                  X5        S S S 5        O[3        U[4        5      (       aR  URC                  5       S:X  a>  [9        U R;                  5       R<                  5         U RE                  U5        S S S 5        O\US:X  a>  [9        U R;                  5       R<                  5         U RG                  U5        S S S 5        OU(       a  U RI                  X5        U RK                  U5        XRL                  ;   a   U RL                  U   RN                  " U 40 UD6  U RQ                  U RR                  US9  g ! , (       d  f       Ni= f! , (       d  f       Nz= f! , (       d  f       N= f! , (       d  f       N= f)Nr+   r   z?`r` should be a positive integer value but the value passed is         )pr   FT)
input_sizeoutput_sizebiasr   skip_bias_addr   config)in_featuresout_featuresr9   dtype)r7   r8   r9   r   r   r;   pissacordaoloraloftq)inference_mode)*localscopyr   r-   r   nnDropoutIdentityr   torchfloat32params_dtyper#   r    r"   r<   Linearr=   ColumnParallelLinearlora_Alora_Bmathsqrtscalingr   r!   str
startswithr
   get_base_layerweight
pissa_init
corda_initlower
olora_init
loftq_initreset_lora_parameters%_move_adapter_to_device_of_base_layerlora_variantr&   set_adapteractive_adapters)r+   r,   r-   r   r   r   r   r   r   r   r   rC   r0   r/   lora_dropout_layerr   lora_alora_bs                     r1   r)   LoraParallelLinear.update_layere   sC     6N6^_`^abcc |(2%#!#l!;!#*<,'01BC',}}$\\33++"3"'& 4 F YY1;L;LSX`e`m`mnFYY4+;+;!RW_d_l_lmF\\66 --+'& 7 F %+L!$*L!)3diil)BDLL&)3DLL&&.l# '--2C2N2Nw2W2W"4#6#6#8#?#?@@ A@)3//4E4P4PQX4Y4Y"4#6#6#8#?#?@@ A@)3//4E4K4K4MQX4X"4#6#6#8#?#?@- A@')"4#6#6#8#?#?@- A@&&|G 	22<@,,,l+00@@--nM) A@ A@ A@ A@s0   O O.O"2O3 
O
O"
O03
Pc           	     V   U R                   " U/UQ70 UD6  UR                  SS 5      nU R                  (       a<  U R                  (       a  U R	                  5         U R
                  " U/UQ70 UD6u  pVXV4$ Ub"  [        U R                  R                   S35      eU R                  (       a  U R
                  " U/UQ70 UD6u  pVXV4$ U R
                  " U/UQ70 UD6u  pVUR                  nU R                   H  nXR                  R                  5       ;  a  M"  U R                  U   n	U R                  U   n
U R                  U   nU R                  U   nU R!                  XR"                  R                  5      nXZ" U	" U" U5      5      5      U-  -   nM     UR%                  U5      nXV4$ )Nadapter_namesz* does not support mixed_batch_forward yet.)_check_forward_argspopdisable_adaptersmergedunmerger   r   r   r   r>   r`   rN   keysrO   r   rR   _cast_input_dtyperV   to)r+   xargsr/   rf   resultr9   torch_result_dtypeactive_adapterrN   rO   dropoutrR   s                r1   forwardLoraParallelLinear.forward   s     4T4V4

?D9   {{??1>t>v>LF& |% & 7 788bcdd[[??1>t>v>LF |  ??1>t>v>LF!'"&"6"6!)9)9);;^4^4++N;,,~6**1mm.A.AB&
);"<w"FF #7 YY12F|r3   c                X   [        X5      nU(       d  gU GH  nX0R                  R                  5       ;   d  M#  U R                  5       nU(       a  UR                  R
                  R                  5       nU R                  U5      nXV-   n[        R                  " U5      R                  5       (       d  [        SU S35      eXTR                  l        O9U R                  U5      nUR                  R
                  U-   UR                  l        U R                  R                  U5        GM     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
Nz1NaNs detected in the merged weights. The adapter z seems to be broken)r   rN   rl   rU   rV   datacloneget_delta_weightrI   isfiniteallr   merged_adaptersappend)r+   
safe_mergerf   rs   r   orig_weightsdelta_weights          r1   mergeLoraParallelLinear.merge   s     0D+N!1!1!33!002
 $.#4#4#9#9#?#?#AL#'#8#8#HL#/#>L >>,7;;==(OP^O__rs  .:%%*#'#8#8#HL-7->->-C-Cl-RJ%%*$$++N;) ,r3   c                   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  a  U R                  R                  5       nXR                  R                  5       ;   a@  U R                  5       R                  nU R                  U5      nU=R                  U-  sl        [        U R                  5      S:  a  M  gg)zG
This method unmerges all merged adapter layers from the base weights.
z Already unmerged. Nothing to do.Nr   )rj   warningswarnlenr}   rh   rN   rl   rU   rV   rz   rx   )r+   rs   rV   r   s       r1   rk   LoraParallelLinear.unmerge  s     {{MM<=$&&'!+!11557N!1!1!33,,.55#44^D|+ $&&'!+r3   c                   U R                   U   R                  R                  nU R                   U   R                  R                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                  U   R                  nU R                   U   R                  nU(       a   UR                  5       nUR                  5       n[        Xe-  U R                  5      U R                  U   -  nU(       ai  UR                  US9nUR                  U5      U R                  U   R                  l        UR                  U5      U R                   U   R                  l        U$ )z
Compute the delta weight for the given adapter.

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
cpu)r>   )rO   rV   devicer>   typerI   float16bfloat16rN   floatr	   r$   rR   rn   rx   )r+   adapterr   r>   cast_to_fp32weight_Aweight_Boutput_tensors           r1   rz   #LoraParallelLinear.get_delta_weight  s,    W%,,33G$++11
 {{e+c%--1G1b5TYTbTbKb;;w'..;;w'..~~'H~~'H!("5t7J7JKdll[bNcc),,5,9M 08{{5/ADKK '',/7{{5/ADKK '',r3   c                *   > [         TU ]  5       nSU-   $ )Nzlora.)r   __repr__)r+   repr   s     r1   r   LoraParallelLinear.__repr__2  s    g }r3   )r%   r    r$   r#   r*   )	r   r   r5   FFTFFF)r,   rS   r-   intr   r   r   r   r$   boolr*   r   r   zUnion[bool, str]r   r   r   r   r.   r   )rC   r   )ro   torch.Tensorrp   r   r/   r   )FN)r   r   rf   zOptional[list[str]]returnNone)r   r   )r   r   )r   rS   )r   
__module____qualname____firstlineno____doc__r   r&   r'   r)   ru   r   rk   rz   r   __static_attributes____classcell__)r   s   @r1   r   r       s     !$(-.2 ;- ;-
 ;- ;- ;- ;- "&;- ,;- ;- ;- ;- ;-J ''$UN UNn<&<P, D r3   r   c                   S n[        U [        5      (       a  U R                  5       nOU nUR                  (       a!  [        R
                  " UR                  5      nOS nU(       a  [        UUR                  R                  UR                  R                  45      (       a  UR                  5       nUR                  n[        U[        5      (       a2  UR                  R                  R                  n	U	" S0 UR                  D6nXS'   US   (       a"  [        R                   " S5        S=US'   Ul        [%        SXUR                  S.UD6nU$ )Nr   r$   zfan_in_fan_out is set to True but the target module is `ColumnParallelLinear` or `RowParallelLinear`. Setting fan_in_fan_out to False.F)r   r,   r     )r!   r   rU   r   	importlibimport_modulemegatron_coretensor_parallelrM   r"   rE   dicttransformertransformer_configTransformerConfigr   r   r$   r   )
targetr,   lora_configr/   
new_moduletarget_base_layerr   megatron_kwargsr   transformer_config_classs
             r1   dispatch_megatronr   7  s5    J&.))"113"""!//0I0IJ		&	&	;	;]=Z=Z=l=lm  !++-%55ot,,'4'@'@'S'S'e'e$6U9T9TUO-<)*+,MM3
 NSRO,-0J' 
-B_B_
cr

 r3   )r   ztorch.nn.Moduler,   rS   r/   r   r   zOptional[torch.nn.Module])
__future__r   r   rP   r   typingr   r   r   rI   torch.nnrF   torch.nn.initr&   peft.tuners.tuners_utilsr   r   
peft.utilsr	   peft.utils.integrationsr
   layerr   Moduler   r   r   r3   r1   <module>r      so    #    ' '     L   5 TI Tn''' 	'
 'r3   