
    oi                    l    S r SSKJr  SSKJr  SSKrSSKJr  SSKJ	r	  / SQr
S rSS jrSS	 jrSS
 jrg)z=Utilities for Orthogonal Subspace Learning with Adaptive OSF.    )annotations)AnyN)nn)decompose_weight_matrix$project_gradient_to_orthogonal_spacereconstruct_weight_matrixc                H    [        U S5      (       a  U R                  5       $ U $ )zHWait for AsyncCollectiveTensor if needed, otherwise return tensor as-is.wait)hasattrr
   )tensors    O/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/osf/utils.py_wait_if_asyncr   #   s     vv{{}M    c                   U R                   nU R                  nU R                  [        R                  5      n[        R
                  R                  USS9u  pVn[        XR                  S   5      nUSS2SU24   R                  5       R                  5       R                  X#S9USU R                  5       R                  5       R                  X#S9USU2SS24   R                  5       R                  5       R                  X#S9[        R                  " USS2US24   R                  5       R                  5       R                  X#S95      [        R                  " XhS R                  5       R                  5       R                  X#S95      [        R                  " XxS2SS24   R                  5       R                  5       R                  X#S95      US.n	U	$ )zJPerform an SVD of ``weight`` and split it into frozen and trainable parts.F)full_matricesr   N)devicedtype)U_highS_highV_highU_lowS_lowV_low	rank_high)r   r   totorchfloat32linalgsvdminshape
contiguousdetachr   	Parameter)
weighttop_kdevice_local
orig_dtypeWUSVtkr   s
             r   r   r   *   s   ==LJ		%-- A||7HA"E771:A ArrE(%%'..033<3ZBQ%""$++-000WRaRU)&&(//144L4[a12h113::<??|?fgae..0779<<L<cdbQi224;;=@@@ghC Jr   c                ~   U S   nU S   nU S   nU S   nU S   nU S   nUR                  5       S:  a<  UR                  5       S:  a(  [        R                  " XR                  S5      -  U5      O=[        R                  " UR                  S5      UR                  S5      UR                  S	9nUR                  5       S:  a<  UR                  5       S:  a(  [        R                  " XER                  S5      -  U5      O=[        R                  " UR                  S5      UR                  S5      UR                  S	9nXx-   $ )
z4Reconstruct a weight matrix from its SVD components.r   r   r   r   r   r   r      )r   )numelr   mm	unsqueezezerossizer   )	svd_dictr   r   r   r   r   r   	high_partlow_parts	            r   r   r   >   s   hFhFhFWEWEWE <<>A&,,.1"4 	**1--v6[[A

1fmmL  ;;=1!2 	++U3[[QQM 
 r   c                  ^^	^
^ U S   R                   c!  U S   R                   c  U S   R                   c  gU S   mU S   m	U S   R                   Gb*  U S   R                   m
[        TSU4S j5      " 5       n[        T
SU
4S	 j5      " 5       n[        R                  " UR	                  S
S5      U5      n[
        R                  " 5       (       a`  [
        R                  " 5       (       aF  [
        R                  " 5       S:  a-  [
        R                  " U[
        R                  R                  S9  UR                  XSS9  [        T
S5      (       a  T
R                  R                  U5        OT
R                  U5        U S   R                   GbA  U S   R                   m[        T	SU	4S j5      " 5       n[        TSU4S j5      " 5       n[        R                  " UR	                  S
S5      U5      n[
        R                  " 5       (       a`  [
        R                  " 5       (       aF  [
        R                  " 5       S:  a-  [
        R                  " U[
        R                  R                  S9  [        R                  " XV5      nUR!                  USS9  [        TS5      (       a  TR                  R                  U5        gTR                  U5        gg)zUProject gradients of ``U_low`` and ``V_low`` to be orthogonal to the high rank space.r   Nr   r   r   r   to_localc                    > T $ N )r   s   r   <lambda>6project_gradient_to_orthogonal_space.<locals>.<lambda>`       6r   c                    > T $ r;   r<   )dUs   r   r=   r>   a       2r   r   r/   )opg      )alpha_local_tensorc                    > T $ r;   r<   )r   s   r   r=   r>   u   r?   r   c                    > T $ r;   r<   )dVs   r   r=   r>   v   rB   r   )gradgetattrr   r1   	transposedistis_availableis_initializedget_world_size
all_reduceReduceOpSUMaddmm_r   rE   copy_add_)r5   local_U_highlocal_dU
proj_coefflocal_V_highlocal_dVG_localupdater   r   rA   rH   s           @@@@r   r   r   T   s/   %(7*;*@*@*HXV]M^McMcMkhFhF )g##vz>BD2z:68 XXl44Q:HE
4#6#6#8#8T=P=P=RUV=VOOJ4==+<+<==2''""8,HHX )g##vz>BD2z:68 ((<11!Q7F4#6#6#8#8T=P=P=RUV=VOOG(9(9: (,fD)2''""8,HHX% *r   )r%   torch.Tensorr&   intreturndict[str, Any])r5   zdict[str, torch.Tensor]r_   r]   )r5   r`   r_   None)__doc__
__future__r   typingr   r   torch.distributeddistributedrL   r   __all__r   r   r   r   r<   r   r   <module>rh      s5    D "     ( ,1r   