
    oi:                         S SK r S SKJrJr  S SKrS SKJr  S SKJr  S SK	J
r
Jr  S SKJr   " S S\
5      r " S S	\R                  \5      r " S
 S\R                  \5      r " S S\5      r " S S\5      rg)    N)AnyOptional)Conv1D)BaseTunerLayercheck_adapters_to_merge)	transposec                   Z    \ rS rSrSrS\R                  S\SS4S jrSS\4S	 jjr	S
 r
Srg)IA3Layer   )ia3_l
base_layeris_feedforwardreturnNc                    Xl         [        R                  " 0 5      U l        SU l        / U l        X l        U R                  5       n[        U[        R                  5      (       a  UR                  UR                  pTO[        U[        R                  [        R                  45      (       a  UR                  UR                  pTO[        U[        R                   5      (       a  UR"                  UR$                  pTOv[        U[&        5      (       aJ  [)        UR*                  S5      (       a  UR*                  R,                  OUR*                  R.                  u  pEO[1        S[3        U5       35      eX@l
        XPl        g )NFds_shapezUnsupported layer type )r   nnParameterDictr   _disable_adaptersmerged_adaptersr   get_base_layer
isinstanceLinearin_featuresout_featuresConv2dConv3din_channelsout_channels	Embeddingnum_embeddingsembedding_dimr   hasattrweightr   shape
ValueErrortype)selfr   r   kwargsr   r   s         O/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/ia3/layer.py__init__IA3Layer.__init__   s   $%%b)
!&!,((*
j")),,(2(>(>
@W@W
RYY		$:;;(2(>(>
@W@W
BLL11(2(A(A:C[C[
F++.5j6G6G.T.T
!!**ZdZkZkZqZq &K 6tJ7G6HIJJ&(    inference_modec                 |   U R                   (       a#  [        R                  " SU R                  45      nO"[        R                  " U R                  S45      n[
        R                  " U5      U R                  U'   U(       a  U R                  U5        U R                  U5        U R                  U R                  US9  g )N   r-   )r   torchrandnr   r   r   	Parameterr   reset_ia3_parameters%_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r'   adapter_nameinit_ia3_weightsr-   r(   r#   s         r)   update_layerIA3Layer.update_layer6   s     [[!T%5%5!67F[[$"3"3Q!78F#%<<#7

< %%l322<@--nMr,   c                     XR                   R                  5       ;   a.  [        R                  R	                  U R                   U   S5        g g )Ng      ?)r   keysr   init	constant_)r'   r8   s     r)   r4   IA3Layer.reset_ia3_parametersC   s5    ::??,,GGdjj6< -r,   )r   r   r   r   r   r   r   F)__name__
__module____qualname____firstlineno__adapter_layer_namesr   Moduleboolr*   r:   r4   __static_attributes__ r,   r)   r
   r
      s:    $)299 )d )QU )0N4 N=r,   r
   c                      ^  \ rS rSr    SS\R
                  S\S\S\S\S\SS	4U 4S
 jjjrSS\S\	\
\      SS	4S jjrSS jrS\R                  S\S\S\R                  4S jrSrU =r$ )r   I   r   r8   fan_in_fan_outr   is_target_conv_1d_layerr9   r   Nc                    > [         TU ]  5         [        R                  XUS9  X0l        XPl        X l        U R                  X&5        g N)r   )superr*   r
   rM   rN   _active_adapterr:   )	r'   r   r8   rM   r   rN   r9   r(   	__class__s	           r)   r*   Linear.__init__K   sC     	$>J,'>$+,9r,   
safe_mergeadapter_namesc                    [        X5      nU(       d  gU GH3  nX0R                  R                  5       ;   d  M#  U R                  5       n[	        U R                  U   R
                  U R                  5      nUR                  R
                  R                  nU(       a  UR                  R
                  n[        R                  " Xu5      n[        R                  " U5      R                  5       (       d  [        SU S35      eUR                  U5      UR                  l        OI[        R                  " UR                  R
                  U5      R                  U5      UR                  l        U R                  (       d  UR                   b  U R                  U   R#                  UR                   R$                  5      nUR                   R
                  R                  n[        R                  " UR                   R
                  UR
                  5      R                  U5      UR                   l        U R&                  R)                  U5        GM6     g)  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`List[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
N1NaNs detected in the merged weights. The adapter  seems to be broken)r   r   r=   r   r   datarM   r#   dtyper1   mulisfiniteallr%   tor   biasreshaper$   r   append)	r'   rU   rV   active_adapterr   r   
orig_dtypeorig_weightsscalings	            r)   mergeLinear.merge\   s    0D+N!22!002
!$**^"<"A"A4CVCVW'..3399
#-#4#4#9#9L#(99\#AL >>,7;;==(OP^O__rs  .:__Z-HJ%%*-2YYz7H7H7M7Mu-U-X-XYc-dJ%%***
0K"jj8@@AVAVWG!+!5!5!;!;J+099Z__5I5I7<<+X+[+[\f+gJOO($$++N;- ,r,   c                 <   U R                   (       d  [        R                  " S5        g[        R                  " S5        [        U R                  5      S:  Ga  U R                  R                  5       nXR                  R                  5       ;   Gao  U R                  5       n[        U R                  U   R                  U R                  5      S-   nUR                  R                  R                  n[        R                  " UR                  R                  U5      R!                  U5      UR                  l
        U R"                  (       d  UR$                  b  U R                  U   R'                  UR$                  R(                  5      nUR$                  R                  R                  n[        R                  " UR$                  R                  UR                  S-   5      R!                  U5      UR$                  l
        [        U R                  5      S:  a  GM  gg)G
This method unmerges all merged adapter layers from the base weights.
 Already unmerged. Nothing to do.N,Unmerge result can be inaccurate for (IA)^3.r   :0yE>)mergedwarningswarnlenr   popr   r=   r   r   r[   rM   r#   r\   r1   divr`   r   ra   rb   r$   )r'   rd   r   r   re   rg   s         r)   unmergeLinear.unmerge   sv    {{MM<=DE$&&'!+!11557N!22!002
!$**^"<"A"A4CVCVWZ^^'..3399
).:3D3D3I3I5)Q)T)TU_)`
!!&**
0K"jj8@@AVAVWG!+!5!5!;!;J+099Z__5I5I7<<Z^K^+_+b+bcm+nJOO( $&&'!++r,   xargsr(   c                    UR                   =pEU R                  (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ SnU R
                   HZ  nXR                  R                  5       ;  a  M"  U R                  U   R                   nXpR                  U   R                  5       -  nM\     U R                  (       a<  UR                  U5      nX-  R                  U5      n	U R                  " U	/UQ70 UD6nU$ U R                  " U/UQ70 UD6nUR                   n
Xg-  R                  U
5      nU$ Nr/   )r\   disable_adaptersro   ru   r   r7   r   r=   flattenr   r`   )r'   rw   rx   r(   r\   previous_dtyperesultia3_scalingrd   intermresult_dtypes              r)   forwardLinear.forward   sM   !"(  {{__Q888F, + [[__Q888F( % K"&"6"6!)::

>288zz.9AACC	 #7 ""DDK /--n=A$A&A 	 <T<V<%|| .22<@r,   )rR   rM   rN   )FFFTFNr   N)rB   rC   rD   rE   r   rG   strrH   r*   r   listrh   ru   r1   Tensorr   r   rI   __classcell__rS   s   @r)   r   r   I   s      %$(-!%:II: : 	:
 : "&: : 
: :"(< (<Xd3i=P (<\` (<To. c S U\\  r,   r   c                      ^  \ rS rSr   SS\R
                  S\S\S\S\SS4U 4S	 jjjrSS
\4S jjr	SS\S\
\\      SS4S jjrSS jrS\R                  S\S\S\R                  4S jrSrU =r$ )_ConvNd   r   r8   rM   r   r9   r   Nc                    > [         TU ]  5         [        R                  XUS9  X0l        X l        UR
                  R                  5       U l        U R                  X%5        g rP   )	rQ   r*   r
   rM   rR   r#   dim_kernel_dimr:   )r'   r   r8   rM   r   r9   r(   rS   s          r)   r*   _ConvNd.__init__   sT     	$>J,+%,,002,9r,   r-   c                 |   U R                   (       a  U R                  OU R                  nSU4SU R                  S-
  -  -   n[        R
                  " U5      n[        R                  " U5      U R                  U'   U(       a  U R                  U5        U R                  U5        U R                  U R                  US9  g )Nr/   )r/      r0   )r   r   r   r   r1   r2   r   r3   r   r4   r5   r6   r7   )r'   r8   r9   r-   r(   num_featuresweights_sizer#   s           r)   r:   _ConvNd.update_layer   s    +/+>+>t''DDUDU<(443C3Ca3G+HH\*#%<<#7

< %%l322<@--nMr,   rU   rV   c                    [        X5      nU(       d  gU GH/  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  R                  nU R                  U   R
                  nU R                  (       d  UR                  SS5      nU(       a  [        R                  " UR                  R
                  U5      R                  5       n[        R                  " U5      R                  5       (       d  [        SU S35      eUR                  U5      UR                  l        OI[        R                  " UR                  R
                  U5      R                  U5      UR                  l        U R                  (       d  UR                   b  U R                  U   R#                  UR                   R$                  5      n[        R                  " UR                   R
                  UR
                  5      R                  U5      UR                   l        U R&                  R)                  U5        GM2     g)rX   Nr   r/   rY   rZ   )r   r   r=   r   r#   r[   r\   r   r   r1   r]   cloner^   r_   r%   r`   ra   rb   r$   r   rc   )	r'   rU   rV   rd   r   re   r   output_weightrg   s	            r)   rh   _ConvNd.merge   s    0D+N!22!002
'..3399
"jj8==**"-"7"71"=K$)IIj.?.?.D.Dk$R$X$X$ZM >>-8<<>>(OP^O__rs  .;-=-=j-IJ%%*-2YYz7H7H7M7M{-[-^-^_i-jJ%%***
0K"jj8@@AVAVWG+099Z__5I5I7<<+X+[+[\f+gJOO($$++N;1 ,r,   c                 T   U R                   (       d  [        R                  " S5        g[        R                  " S5        [        U R                  5      S:  Ga  U R                  R                  5       nXR                  R                  5       ;   Ga{  U R                  5       nUR                  R                  R                  nU R                  U   R                  nU R                  (       d  UR                  SS5      n[        R                  " UR                  R                  US-   5      R!                  U5      UR                  l
        U R                  (       d  UR"                  b  U R                  U   R%                  UR"                  R&                  5      nUR"                  R                  R                  n[        R(                  " UR"                  R                  UR                  5      R!                  U5      UR"                  l
        [        U R                  5      S:  a  GM  gg)rk   rl   Nrm   r   r/   rn   )ro   rp   rq   rr   r   rs   r   r=   r   r#   r[   r\   r   r   r1   rt   r`   ra   rb   r$   r]   )r'   rd   r   re   r   rg   s         r)   ru   _ConvNd.unmerge  s~    {{MM<=DE$&&'!+!11557N!22!002
'..3399
"jj8==**"-"7"71"=K).:3D3D3I3I;Y]K])^)a)abl)m
!!&**
0K"jj8@@AVAVWG!+!5!5!;!;J+099Z__5I5I7<<+X+[+[\f+gJOO( $&&'!++r,   rw   rx   r(   c                    UR                   =pEU R                  (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nGOU R                  (       a  U R                  " U/UQ70 UD6nOSnU R
                   HL  nXR                  R                  5       ;  a  M"  U R                  U   R                   nXpR                  U   -  nMN     U R                  (       a]  UR                  U5      nX-  R                  U R                  5       R                  R                   5      n	U R                  " U	/UQ70 UD6nO*U R                  " U/UQ70 UD6nUR                  U5      U-  nUR                  U5      nU$ rz   )r\   r{   ro   ru   r   r7   r   r=   r   r`   r   r#   )
r'   rw   rx   r(   r\   r}   r~   r   rd   r   s
             r)   r   _ConvNd.forward  sC   !"(  {{__Q888F[[__Q888FK"&"6"6!)::

>288zz.99	 #7 ""DDK /--d.A.A.C.J.J.P.PQA$A&A<T<V<5)K7>*r,   )rR   r   rM   )FFTrA   r   r   )rB   rC   rD   rE   r   rG   r   rH   r*   r:   r   r   rh   ru   r1   r   r   r   rI   r   r   s   @r)   r   r      s    
  %$!%:II: : 	:
 : : 
: :"	N4 	N*< *<Xd3i=P *<\` *<Xh2 c S U\\  r,   r   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )r   i<  c                 v   > [         TU ]  " U0 UD6  U R                  S:X  d  [        SU R                   35      eg )N   z0Conv2d layer kernel must have 4 dimensions, not rQ   r*   r   r%   r'   rx   r(   rS   s      r)   r*   Conv2d.__init__?  C    $)&)1$OPTP`P`Oabcc %r,   rJ   rB   rC   rD   rE   r*   rI   r   r   s   @r)   r   r   <  s    d dr,   r   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )r   iE  c                 v   > [         TU ]  " U0 UD6  U R                  S:X  d  [        SU R                   35      eg )N   z0Conv2d layer kernel must have 5 dimensions, not r   r   s      r)   r*   Conv3d.__init__G  r   r,   rJ   r   r   s   @r)   r   r   E  s    d dr,   r   )rp   typingr   r   r1   torch.nnr   transformers.pytorch_utilsr   peft.tuners.tuners_utilsr   r   
peft.utilsr   r
   rG   r   r   r   r   rJ   r,   r)   <module>r      so         - L  ,=~ ,=^oRYY od~bii ~BdW ddW dr,   