
    +h(`                        S SK r S SKJrJrJr  S SKrS SKJr  S SKJs  J	r
  SSKJrJr  SSKJr  SSKJrJr   " S S	\R&                  5      r " S
 S\R*                  5      r " S S\R&                  5      r " S S\R&                  5      r " S S\R&                  5      r " S S\R&                  5      r " S S\R&                  5      r " S S\R&                  5      r " S S\R&                  5      r " S S\R&                  5      r " S S\R&                  5      r " S S\R&                  5      r \" S S!5      (       a  \R*                  rO " S" S#\R&                  5      r " S$ S%\R&                  5      r! " S& S'\R&                  5      r" " S( S)\R&                  5      r# " S* S+\R&                  5      r$     S3S,\%S-\\&   S.\'S/\(S0\(S1\R&                  4S2 jjr)g)4    N)DictOptionalTuple   )is_torch_npu_availableis_torch_version   )get_activation)CombinedTimestepLabelEmbeddings)PixArtAlphaCombinedTimestepSizeEmbeddingsc                      ^  \ rS rSrSr     SS\S\\   S\\   S\S\S\4U 4S	 jjjr	 SS
\
R                  S\\
R                     S\\
R                     S\
R                  4S jjrSrU =r$ )AdaLayerNorm   a  
Norm layer modified to incorporate timestep embeddings.

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`, *optional*): The size of the embeddings dictionary.
    output_dim (`int`, *optional*):
    norm_elementwise_affine (`bool`, defaults to `False):
    norm_eps (`bool`, defaults to `False`):
    chunk_dim (`int`, defaults to `0`):
embedding_dimnum_embeddings
output_dimnorm_elementwise_affinenorm_eps	chunk_dimc                 >  > [         TU ]  5         X`l        U=(       d    US-  nUb  [        R                  " X!5      U l        OS U l        [        R                  " 5       U l        [        R                  " X5      U l	        [        R                  " US-  XT5      U l        g Nr   )super__init__r   nn	EmbeddingembSiLUsiluLinearlinear	LayerNormnorm)selfr   r   r   r   r   r   	__class__s          X/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/models/normalization.pyr   AdaLayerNorm.__init__)   st     	"4=1#4
%||NBDHDHGGI	ii:LLq(T	    xtimesteptembreturnc                 P   U R                   b  U R                  U5      nU R                  U R                  U5      5      nU R                  S:X  a+  UR	                  SSS9u  pEUS S 2S S S 24   nUS S 2S S S 24   nOUR	                  SSS9u  pTU R                  U5      SU-   -  U-   nU$ )Nr	   r   dimr   )r   r    r   r   chunkr"   )r#   r(   r)   r*   shiftscales         r%   forwardAdaLayerNorm.forward@   s     8888H%D{{499T?+>>Q  ::aQ:/LE!T1*%E!T1*%E::aQ:/LEIIaLAI&.r'   )r   r   r    r"   r   )NNFh㈵>r   )NN)__name__
__module____qualname____firstlineno____doc__intr   boolfloatr   torchTensorr2   __static_attributes____classcell__r$   s   @r%   r   r      s    
 )-$((-UU !U SM	U
 "&U U U U0 hl)1%,,)?NVW\WcWcNd	 r'   r   c                   N    \ rS rSrS\R
                  S\R
                  4S jrSrg)FP32LayerNormU   inputsr+   c                 L   UR                   n[        R                  " UR                  5       U R                  U R
                  b  U R
                  R                  5       OS U R                  b  U R                  R                  5       OS U R                  5      R                  U5      $ N)	dtypeF
layer_normr<   normalized_shapeweightbiasepsto)r#   rE   origin_dtypes      r%   r2   FP32LayerNorm.forwardV   su    ||||LLN!!#';;#:DKK!%!6DIIOODHH
 "\
	r'    N)r5   r6   r7   r8   r=   r>   r2   r?   rR   r'   r%   rC   rC   U   s    ell u|| r'   rC   c            	          ^  \ rS rSrSrSS\S\S\SS4U 4S jjjr SS	\	R                  S
\\	R                     S\\	R                  S4   4S jjrSrU =r$ )SD35AdaLayerNormZeroXa   z
Norm layer adaptive layer norm zero (AdaLN-Zero).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`): The size of the embeddings dictionary.
r   	norm_typerM   r+   Nc                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  US9U l        US:X  a  [        R                  " USSS9U l        g [        SU S35      e)	N	   rM   rJ   Fư>elementwise_affinerN   Unsupported `norm_type` (z-) provided. Supported ones are: 'layer_norm'.
r   r   r   r   r   r   r    r!   r"   
ValueErrorr#   r   rV   rM   r$   s       r%   r   SD35AdaLayerNormZeroX.__init__j   sg    GGI	iiq=/@tL$]uRVWDI8Cpqrrr'   hidden_statesr   .c           	          U R                  U R                  U5      5      nUR                  SSS9u	  p4pVpxpnU R                  U5      nUSUS S 2S 4   -   -  US S 2S 4   -   nUSU
S S 2S 4   -   -  U	S S 2S 4   -   nXXgXU4$ )NrX   r	   r-   r    r   r/   r"   )r#   rb   r   	shift_msa	scale_msagate_msa	shift_mlp	scale_mlpgate_mlp
shift_msa2
scale_msa2	gate_msa2norm_hidden_statesnorm_hidden_states2s                 r%   r2   SD35AdaLayerNormZeroX.forwardt   s    
 kk$))C.)lolulu1 mv m
i	h9
`i "YY}5*a)AtG2D.DE	RSUYRYHZZ0A
1d78K4KLzZ[]aZaObb	h]fffr'   r    r"   r   rJ   TrG   )r5   r6   r7   r8   r9   r:   strr;   r   r=   r>   r   r   r2   r?   r@   rA   s   @r%   rT   rT   a   s    sc sc sPT s`d s s '+g||g ell#g 
u||S 	!	g gr'   rT   c                   `  ^  \ rS rSrSrSS\S\\   4U 4S jjjr    SS\R                  S\\R                     S\\R                     S	\\R                     S
\\R                     S\\R                  \R                  \R                  \R                  \R                  4   4S jjrSrU =r$ )AdaLayerNormZero   
Norm layer adaptive layer norm zero (adaLN-Zero).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`): The size of the embeddings dictionary.
r   r   c                 V  > [         TU ]  5         Ub  [        X!5      U l        OS U l        [        R
                  " 5       U l        [        R                  " USU-  US9U l        US:X  a  [        R                  " USSS9U l
        g US:X  a  [        USSS9U l
        g [        S	U S
35      e)N   rY   rJ   FrZ   r[   fp32_layer_norm)r\   rM   r]   @) provided. Supported ones are: 'layer_norm', 'fp32_layer_norm'.)r   r   r   r   r   r   r   r   r    r!   r"   rC   r_   )r#   r   r   rV   rM   r$   s        r%   r   AdaLayerNormZero.__init__   s    %6~UDHDHGGI	iiq=/@tL$]uRVWDI++%mTYZDI+I;6vw r'   r(   r)   class_labelshidden_dtyper   r+   c                     U R                   b  U R                  X#US9nU R                  U R                  U5      5      nUR                  SSS9u  pgppU R	                  U5      SUS S 2S 4   -   -  US S 2S 4   -   nXXU4$ )N)r~   ry   r	   r-   )r   r    r   r/   r"   )r#   r(   r)   r}   r~   r   re   rf   rg   rh   ri   rj   s               r%   r2   AdaLayerNormZero.forward   s     88((8(MCkk$))C.)ILSTZ[I\F	h9IIaLA	!T' 223i46HHI(::r'   )r   r    r"   r   )NrJ   T)NNNN)r5   r6   r7   r8   r9   r:   r   r   r=   r>   
LongTensorrH   r   r2   r?   r@   rA   s   @r%   ru   ru      s    c 8C=  * ,037.2&*;<<; 5<<(; u//0	;
 u{{+; ell#; 
u||U\\5<<u||S	T; ;r'   ru   c                      ^  \ rS rSrSrS
S\4U 4S jjjr SS\R                  S\	\R                     S\
\R                  \R                  \R                  \R                  \R                  4   4S jjrS	rU =r$ )AdaLayerNormZeroSingle   rw   r   c                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  US9U l        US:X  a  [        R                  " USSS9U l        g [        SU S35      e)	N   rY   rJ   FrZ   r[   r]   r{   r^   r`   s       r%   r   AdaLayerNormZeroSingle.__init__   sk    GGI	iiq=/@tL$]uRVWDI+I;6vw r'   r(   r   r+   c                     U R                  U R                  U5      5      nUR                  SSS9u  p4nU R                  U5      SUS S 2S 4   -   -  US S 2S 4   -   nX4$ )Nr   r	   r-   rd   )r#   r(   r   re   rf   rg   s         r%   r2   AdaLayerNormZeroSingle.forward   sg    
 kk$))C.)),1!)<&	hIIaLA	!T' 223i46HH{r'   rq   rr   rG   r5   r6   r7   r8   r9   r:   r   r=   r>   r   r   r2   r?   r@   rA   s   @r%   r   r      su    
c 
 
 '+<< ell# 
u||U\\5<<u||S	T	 r'   r   c                      ^  \ rS rSrSrS\S\S\4U 4S jjr SS\	R                  S\\	R                     S	\\	R                  \	R                  \	R                  \	R                  4   4S
 jjrSrU =r$ )LuminaRMSNormZero   zx
Norm layer adaptive RMS normalization zero.

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
r   r   r   c                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " [        US5      SU-  SS9U l        [        XS9U l	        g )Ni      TrY   rN   )
r   r   r   r   r   r   minr    RMSNormr"   )r#   r   r   r   r$   s       r%   r   LuminaRMSNormZero.__init__   sN    GGI	iit$

 M8	r'   r(   r   r+   c                     U R                  U R                  U5      5      nUR                  SSS9u  p4pVU R                  U5      SUS S 2S 4   -   -  nXXV4$ )Nr   r	   r-   rd   )r#   r(   r   rf   rg   ri   rj   s          r%   r2   LuminaRMSNormZero.forward   s\    
 kk$))C.)3699QA93F0	YIIaLA	!T' 223I//r'   rq   rG   )r5   r6   r7   r8   r9   r:   r<   r;   r   r=   r>   r   r   r2   r?   r@   rA   s   @r%   r   r      sy    9c 9U 9UY 9 '+	0<<	0 ell#	0 
u||U\\5<<E	F		0 	0r'   r   c                   0  ^  \ rS rSrSrSS\S\4U 4S jjjr   SS\R                  S\
\\\R                  4      S\
\   S	\
\R                     S
\\R                  \R                  \R                  \R                  \R                  4   4
S jjrSrU =r$ )AdaLayerNormSingle   a<  
Norm layer adaptive layer norm single (adaLN-single).

As proposed in PixArt-Alpha (see: https://huggingface.co/papers/2310.00426; Section 2.3).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    use_additional_conditions (`bool`): To use additional conditions for normalization or not.
r   use_additional_conditionsc                    > [         TU ]  5         [        XS-  US9U l        [        R
                  " 5       U l        [        R                  " USU-  SS9U l        g )Nr   )size_emb_dimr   ry   TrY   )	r   r   r   r   r   r   r   r   r    )r#   r   r   r$   s      r%   r   AdaLayerNormSingle.__init__   sM    <(:Vo
 GGI	iiq=/@tLr'   r)   added_cond_kwargs
batch_sizer~   r+   c                     U=(       d    S S S.nU R                   " U40 UDX4S.D6nU R                  U R                  U5      5      U4$ )N)
resolutionaspect_ratio)r   r~   r   r    r   )r#   r)   r   r   r~   embedded_timesteps         r%   r2   AdaLayerNormSingle.forward  sN     .[VZ1[ HHXu1Buzu{{499%678:KKKr'   r   )F)NNN)r5   r6   r7   r8   r9   r:   r;   r   r=   r>   r   r   rs   rH   r   r2   r?   r@   rA   s   @r%   r   r      s    Mc Md M M @D$(.2
L,,
L $Dell):$;<
L SM	
L
 u{{+
L 
u||U\\5<<u||S	T
L 
Lr'   r   c                      ^  \ rS rSrSr SS\S\S\S\\   S\4
U 4S jjjr	S	\
R                  S
\
R                  S\
R                  4S jrSrU =r$ )AdaGroupNormi  a  
GroupNorm layer modified to incorporate timestep embeddings.

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`): The size of the embeddings dictionary.
    num_groups (`int`): The number of groups to separate the channels into.
    act_fn (`str`, *optional*, defaults to `None`): The activation function to use.
    eps (`float`, *optional*, defaults to `1e-5`): The epsilon value to use for numerical stability.
r   out_dim
num_groupsact_fnrN   c                    > [         TU ]  5         X0l        XPl        Uc  S U l        O[        U5      U l        [        R                  " XS-  5      U l        g r   )	r   r   r   rN   actr
   r   r   r    )r#   r   r   r   r   rN   r$   s         r%   r   AdaGroupNorm.__init__  sF     	$>DH%f-DHii{;r'   r(   r   r+   c                    U R                   (       a  U R                  U5      nU R                  U5      nUS S 2S S 2S S 4   nUR                  SSS9u  p4[        R                  " XR
                  U R                  S9nUSU-   -  U-   nU$ )Nr   r	   r-   r   )r   r    r/   rI   
group_normr   rN   )r#   r(   r   r1   r0   s        r%   r2   AdaGroupNorm.forward(  sz    88((3-Ckk#!Qd"#yyy*LLOO:UOe#r'   )r   rN   r    r   )Nr4   )r5   r6   r7   r8   r9   r:   r   rs   r<   r   r=   r>   r2   r?   r@   rA   s   @r%   r   r     sq    	 mq< <+.<<?<IQRU<di< <	 	ELL 	U\\ 	 	r'   r   c                      ^  \ rS rSrSr    SS\S\4U 4S jjjrS\R                  S\R                  S\R                  4S	 jr	S
r
U =r$ )AdaLayerNormContinuousi4  aq  
Adaptive normalization layer with a norm layer (layer_norm or rms_norm).

Args:
    embedding_dim (`int`): Embedding dimension to use during projection.
    conditioning_embedding_dim (`int`): Dimension of the input condition.
    elementwise_affine (`bool`, defaults to `True`):
        Boolean flag to denote if affine transformation should be applied.
    eps (`float`, defaults to 1e-5): Epsilon factor.
    bias (`bias`, defaults to `True`): Boolean flag to denote if bias should be use.
    norm_type (`str`, defaults to `"layer_norm"`):
        Normalization layer to use. Values supported: "layer_norm", "rms_norm".
r   conditioning_embedding_dimc                 
  > [         TU ]  5         [        R                  " 5       U l        [        R
                  " X!S-  US9U l        US:X  a  [        XX55      U l        g US:X  a  [        XU5      U l        g [        SU 35      e)Nr   rY   rJ   rms_normunknown norm_type )r   r   r   r   r   r   r    r!   r"   r   r_   )r#   r   r   r\   rN   rM   rV   r$   s          r%   r   AdaLayerNormContinuous.__init__C  sw     	GGI	ii :A<MTXY$!-6HODI*$4FGDI1)=>>r'   r(   conditioning_embeddingr+   c                    U R                  U R                  U5      R                  UR                  5      5      n[        R
                  " USSS9u  pEU R                  U5      SU-   S S 2S S S 24   -  US S 2S S S 24   -   nU$ )Nr   r	   r-   )r    r   rO   rH   r=   r/   r"   )r#   r(   r   r   r1   r0   s         r%   r2   AdaLayerNormContinuous.forward[  ss    kk$))$:;>>qwwGH{{3q1IIaLAIq$z22U1dA:5FFr'   rq   )Tr4   TrJ   )r5   r6   r7   r8   r9   r:   r   r=   r>   r2   r?   r@   rA   s   @r%   r   r   4  s`    .  ?? %(? ?0 u|| PUP\P\  r'   r   c                      ^  \ rS rSr     SS\S\S\\   4U 4S jjjrS\R                  S\R                  S\R                  4S	 jr	S
r
U =r$ )LuminaLayerNormContinuousic  r   r   r   c                 L  > [         TU ]  5         [        R                  " 5       U l        [        R
                  " X!US9U l        US:X  a  [        XX55      U l        O$US:X  a  [        XUS9U l        O[        SU 35      eS U l        Ub  [        R
                  " XUS9U l        g g )NrY   rJ   r   rN   r\   r   )r   r   r   r   r   r   linear_1r!   r"   r   r_   linear_2)	r#   r   r   r\   rN   rM   rV   r   r$   s	           r%   r   "LuminaLayerNormContinuous.__init__d  s     	 GGI			"<RVW$!-6HODI*$K]^DI1)=>>IIm4HDM r'   r(   r   r+   c                     U R                  U R                  U5      R                  UR                  5      5      nUnU R	                  U5      SU-   S S 2S S S 24   -  nU R
                  b  U R                  U5      nU$ Nr	   )r   r   rO   rH   r"   r   )r#   r(   r   r   r1   s        r%   r2   !LuminaLayerNormContinuous.forward  sn     mmDII&<=@@IJIIaLAIq$z22==$a Ar'   )r   r   r"   r   )Tr4   TrJ   N)r5   r6   r7   r8   r:   r   r   r=   r>   r2   r?   r@   rA   s   @r%   r   r   c  sv      !%II %(I #I I@<< !& 
	 r'   r   c                     ^  \ rS rSrSrS\S\4U 4S jjr SS\R                  S\R                  S\	\R                     S	\
\R                  \R                  \R                  \R                  \R                  4   4S
 jjrSrU =r$ )%CogView3PlusAdaLayerNormZeroTextImagei  rw   r   r.   c                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  SS9U l        [        R                  " USSS9U l        [        R                  " USSS9U l	        g )N   TrY   Fr4   r[   )
r   r   r   r   r   r   r    r!   norm_xnorm_c)r#   r   r.   r$   s      r%   r   .CogView3PlusAdaLayerNormZeroTextImage.__init__  s[    GGI	iirCxdCll35dKll35dKr'   r(   contextr   r+   c                 2   U R                  U R                  U5      5      nUR                  SSS9u  nnnnnn	n
nnnnnU R                  U5      nU R	                  U5      nUSUS S 2S 4   -   -  US S 2S 4   -   nUSUS S 2S 4   -   -  U
S S 2S 4   -   nXXxXXX4
$ )Nr   r	   r-   )r    r   r/   r   r   )r#   r(   r   r   re   rf   rg   rh   ri   rj   c_shift_msac_scale_msa
c_gate_msac_shift_mlpc_scale_mlp
c_gate_mlpnormed_xnormed_contexts                     r%   r2   -CogView3PlusAdaLayerNormZeroTextImage.forward  s     kk$))C.) IIbaI 	
;;q>W-Iag../)AtG2DD AAtG(<$<=AtG@TTI(Z^iuur'   )r    r   r   r   rG   r   rA   s   @r%   r   r     s    Lc L L '+	v<<v v ell#	v
 
u||U\\5<<u||S	Tv vr'   r   c                      ^  \ rS rSr   SS\S\S\S\S\SS4U 4S	 jjjrS
\R                  S\R                  S\R                  S\
\R                  \R                  4   4S jrSrU =r$ )CogVideoXLayerNormZeroi  conditioning_dimr   r\   rN   rM   r+   Nc                    > [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  US9U l        [        R                  " X$US9U l        g )Nry   rY   r   )	r   r   r   r   r   r   r    r!   r"   )r#   r   r   r\   rN   rM   r$   s         r%   r   CogVideoXLayerNormZero.__init__  sJ     	GGI	ii 0!m2C$OLLL^_	r'   rb   encoder_hidden_statesr*   c                 P   U R                  U R                  U5      5      R                  SSS9u  pEpgpU R                  U5      SU-   S S 2S S S 24   -  US S 2S S S 24   -   nU R                  U5      SU-   S S 2S S S 24   -  US S 2S S S 24   -   nXUS S 2S S S 24   U	S S 2S S S 24   4$ )Nry   r	   r-   rd   )
r#   rb   r   r*   r0   r1   gate	enc_shift	enc_scaleenc_gates
             r%   r2   CogVideoXLayerNormZero.forward  s     >B[[SW=Y=_=_`agh=_=i:dy		-0AIq$z3JJUSTVZ\]S]M^^ $		*? @A	MSTVZ\]S]C^ ^ajklnrtukuav vT!T1*5ExPQSWYZPZG[[[r'   rq   )Tr4   T)r5   r6   r7   r8   r:   r;   r<   r   r=   r>   r   r2   r?   r@   rA   s   @r%   r   r     s    
 $(`` ` !	`
 ` ` 
` `\"\\\BG,,\V[VbVb\	u||U\\)	*\ \r'   r   z>=z2.1.0c                   F   ^  \ rS rSrSrS	S\S\S\4U 4S jjjrS rSr	U =r
$ )
r!   i  ap  
LayerNorm with the bias parameter.

Args:
    dim (`int`): Dimensionality to use for the parameters.
    eps (`float`, defaults to 1e-5): Epsilon factor.
    elementwise_affine (`bool`, defaults to `True`):
        Boolean flag to denote if affine transformation should be applied.
    bias (`bias`, defaults to `True`): Boolean flag to denote if bias should be use.
rN   r\   rM   c                   > [         TU ]  5         X l        [        U[        R
                  5      (       a  U4n[        R                  " U5      U l        U(       ah  [        R                  " [        R                  " U5      5      U l        U(       a*  [        R                  " [        R                  " U5      5      OS U l        g S U l        S U l        g rG   )r   r   rN   
isinstancenumbersIntegralr=   Sizer.   r   	ParameteronesrL   zerosrM   r#   r.   rN   r\   rM   r$   s        r%   r   LayerNorm.__init__  s    GH#w//00fzz#DH! ll5::c?;>BBLLS)9:	" 	r'   c                     [         R                  " XR                  U R                  U R                  U R
                  5      $ rG   )rI   rJ   r.   rL   rM   rN   )r#   inputs     r%   r2   LayerNorm.forward  s'    <<xxdiiRRr'   )rM   r.   rN   rL   )r4   TTr5   r6   r7   r8   r9   r<   r;   r   r2   r?   r@   rA   s   @r%   r!   r!     s4    			!U 	!t 	!Z^ 	! 	!"	S 	Sr'   r!   c                   F   ^  \ rS rSrSrS	S\S\S\4U 4S jjjrS rSr	U =r
$ )
r   i  a  
RMS Norm as introduced in https://huggingface.co/papers/1910.07467 by Zhang et al.

Args:
    dim (`int`): Number of dimensions to use for `weights`. Only effective when `elementwise_affine` is True.
    eps (`float`): Small value to use when calculating the reciprocal of the square-root.
    elementwise_affine (`bool`, defaults to `True`):
        Boolean flag to denote if affine transformation should be applied.
    bias (`bool`, defaults to False): If also training the `bias` param.
rN   r\   rM   c                   > [         TU ]  5         X l        X0l        [	        U[
        R                  5      (       a  U4n[        R                  " U5      U l	        S U l
        S U l        U(       ag  [        R                  " [        R                  " U5      5      U l
        U(       a0  [        R                  " [        R                  " U5      5      U l        g g g rG   )r   r   rN   r\   r   r   r   r=   r   r.   rL   rM   r   r   r   r   r   s        r%   r   RMSNorm.__init__  s    "4c7++,,&C::c?	,,uzz#7DKLLS)9:	  r'   c                    [        5       (       a  SS KnU R                  b]  U R                  R                  [        R
                  [        R                  4;   a%  UR                  U R                  R                  5      nUR                  XR                  U R                  S9S   nU R                  b  XR                  -   nU$ UR                  nUR                  [        R                  5      R                  S5      R                  SSS9nU[        R                  " X@R                  -   5      -  nU R                  b  U R                  R                  [        R
                  [        R                  4;   a%  UR                  U R                  R                  5      nXR                  -  nU R                  b  XR                  -   nU$ UR                  U5      nU$ )Nr   )epsilonr   Tkeepdim)r   	torch_npurL   rH   r=   float16bfloat16rO   npu_rms_normrN   rM   float32powmeanrsqrt)r#   rb   r   input_dtypevariances        r%   r2   RMSNorm.forward  sp   !##{{&;;$$(GG$1$4$4T[[5F5F$GM%22=++W[W_W_2`abcMyy$ -		 9   (--K$''6::1=BB2tBTH)EKK888K,LLM{{&;;$$(GG$1$4$4T[[5F5F$GM - ;99($1II$=M  !. 0 0 =r'   )rM   r.   r\   rN   rL   )TFr   rA   s   @r%   r   r     s1    	; ;D ;t ; ;& r'   r   c                   >   ^  \ rS rSrSS\S\4U 4S jjjrS rSrU =r	$ )MochiRMSNormi=  rN   r\   c                 $  > [         TU ]  5         X l        [        U[        R
                  5      (       a  U4n[        R                  " U5      U l        U(       a0  [        R                  " [        R                  " U5      5      U l        g S U l        g rG   )r   r   rN   r   r   r   r=   r   r.   r   r   r   rL   )r#   r.   rN   r\   r$   s       r%   r   MochiRMSNorm.__init__>  s^    c7++,,&C::c?,,uzz#7DKDKr'   c                 8   UR                   nUR                  [        R                  5      R	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  b  XR                  -  nUR                  U5      nU$ )Nr   r   Tr   )	rH   rO   r=   r   r   r   r   rN   rL   )r#   rb   r  r  s       r%   r2   MochiRMSNorm.forwardM  s    #)) ##EMM266q9>>r4>P%Hxx4G(HH;;")KK7M%((5r'   )r.   rN   rL   )T)
r5   r6   r7   r8   r<   r;   r   r2   r?   r@   rA   s   @r%   r  r  =  s%     D  	 	r'   r  c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )GlobalResponseNormiY  z
Global response normalization as introduced in ConvNeXt-v2 (https://huggingface.co/papers/2301.00808).

Args:
    dim (`int`): Number of dimensions to use for the `gamma` and `beta`.
c                    > [         TU ]  5         [        R                  " [        R
                  " SSSU5      5      U l        [        R                  " [        R
                  " SSSU5      5      U l        g r   )r   r   r   r   r=   r   gammabeta)r#   r.   r$   s     r%   r   GlobalResponseNorm.__init__b  sL    \\%++aAs";<
LLQ1c!:;	r'   c                     [         R                  " USSSS9nX"R                  SSS9S-   -  nU R                  X-  -  U R                  -   U-   $ )Nr   )r	   r   T)pr.   r   r   )r.   r   rZ   )r=   r"   r   r  r  )r#   r(   gxnxs       r%   r2   GlobalResponseNorm.forwardg  sO    ZZQFD977r470478zzQV$tyy0144r'   )r  r  )	r5   r6   r7   r8   r9   r   r2   r?   r@   rA   s   @r%   r  r  Y  s    <
5 5r'   r  c                   v   ^  \ rS rSrS
S\S\S\4U 4S jjjrS\R                  S\R                  4S jr	S	r
U =r$ )LpNormim  r  r.   rN   c                 F   > [         TU ]  5         Xl        X l        X0l        g rG   )r   r   r  r.   rN   )r#   r  r.   rN   r$   s       r%   r   LpNorm.__init__n  s    r'   rb   r+   c                 j    [         R                  " XR                  U R                  U R                  S9$ )N)r  r.   rN   )rI   	normalizer  r.   rN   )r#   rb   s     r%   r2   LpNorm.forwardu  s!    {{=FFdhhOOr'   )r.   rN   r  )r   r   g-q=)r5   r6   r7   r8   r:   r<   r   r=   r>   r2   r?   r@   rA   s   @r%   r  r  m  sF    #  u  PU\\ Pell P Pr'   r  rV   num_featuresrN   r\   rM   r+   c                     U S:X  a  [        XX4S9nU$ U S:X  a  [        R                  " XX4S9nU$ U S:X  a  [        R                  " XUS9nU$ [	        SU < S35      e)Nr   )rN   r\   rM   rJ   
batch_norm)rN   affinez
norm_type=z is not supported.)r   r   r!   BatchNorm2dr_   )rV   r  rN   r\   rM   r"   s         r%   get_normalizationr!  y  s{     J|AS_ K 
l	"||LFXd
 K	 
l	"~~l<NO K JI<'9:;;r'   )r  Nr4   TT)*r   typingr   r   r   r=   torch.nnr   torch.nn.functional
functionalrI   utilsr   r   activationsr
   
embeddingsr   r   Moduler   r!   rC   rT   ru   r   r   r   r   r   r   r   r   r   r  r  r  rs   r:   r<   r;   r!  rR   r'   r%   <module>r*     s     ( (     < ' b6299 6r	BLL 	gBII gD(;ryy (;VRYY @0		 0<L LD#299 #L,RYY ,^.		 .b*vBII *vZ\RYY \0 D'""ISBII SB9bii 9|299 85 5(	PRYY 	P ""&#3- 
 	
  YYr'   