
    +h_                     T   S SK JrJr  S SKrS SKrS SKJr  S SKJs  J	r
  SSKJrJr  SSKJr  SSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  \" 5       (       a  S SKJr   " S S\R@                  5      r! " S S\R@                  5      r" " S S\R@                  5      r# " S S\R@                  5      r$ " S S\R@                  5      r% " S S5      r& " S S\R@                  5      r' " S S\R@                  5      r( " S S \R@                  5      r) " S! S"\\\5      r*g)#    )OptionalTupleN   )ConfigMixinregister_to_config)FromOriginalModelMixin)is_torchvision_available   )FeedForward)	Attention)	Timesteps)Transformer2DModelOutput)
ModelMixin)RMSNorm)
transformsc                      ^  \ rS rSr SS\S\S\\\\4   S\SS4
U 4S jjjrS	\R                  S\R                  4S
 jr
SrU =r$ )CosmosPatchEmbed%   in_channelsout_channels
patch_sizebiasreturnNc                    > [         TU ]  5         X0l        [        R                  " XS   -  US   -  US   -  X$S9U l        g )Nr      r
   r   )super__init__r   nnLinearproj)selfr   r   r   r   	__class__s        j/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/models/transformers/transformer_cosmos.pyr   CosmosPatchEmbed.__init__&   sB     	$IIkqM9JqMIJWXMY[gs	    hidden_statesc           
          UR                   u  p#pEnU R                  u  pxn	UR                  X#XG-  XuU-  XU	-  U	5      nUR                  SSSSSSSS5      R	                  SS5      nU R                  U5      nU$ )	Nr   r
         r   r         )shaper   reshapepermuteflattenr!   )
r"   r'   
batch_sizenum_channels
num_framesheightwidthp_tp_hp_ws
             r$   forwardCosmosPatchEmbed.forward.   s    >K>Q>Q;
*e#%--j&7mS[^R^`c
 &--aAq!Q1EMMaQRS		-0r&   )r   r!   )T)__name__
__module____qualname____firstlineno__intr   boolr   torchTensorr9   __static_attributes____classcell__r#   s   @r$   r   r   %   sj    bftt.1t?DS#s]?St[_t	t tU\\ ell  r&   r   c                   r   ^  \ rS rSrS\S\SS4U 4S jjrS\R                  S\R                  4S jrS	r	U =r
$ )
CosmosTimestepEmbedding9   in_featuresout_featuresr   Nc                    > [         TU ]  5         [        R                  " XSS9U l        [        R
                  " 5       U l        [        R                  " USU-  SS9U l        g )NFr   r   )r   r   r   r    linear_1SiLU
activationlinear_2)r"   rI   rJ   r#   s      r$   r    CosmosTimestepEmbedding.__init__:   sF    		+%H'')		,L0@uMr&   	timestepsc                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ N)rL   rN   rO   )r"   rQ   embs      r$   r9   CosmosTimestepEmbedding.forward@   s1    mmI&ooc"mmC 
r&   )rN   rL   rO   )r;   r<   r=   r>   r?   r   rA   rB   r9   rC   rD   rE   s   @r$   rG   rG   9   sA    NC Ns Nt N %,,  r&   rG   c                      ^  \ rS rSrS\S\SS4U 4S jjrS\R                  S\R                  S\R                  4S	 jr	S
r
U =r$ )CosmosEmbeddingG   embedding_dimcondition_dimr   Nc                    > [         TU ]  5         [        USSS9U l        [	        X5      U l        [        USSS9U l        g )NT        )flip_sin_to_cosdownscale_freq_shiftư>)epselementwise_affine)r   r   r   	time_projrG   
t_embedderr   norm)r"   rY   rZ   r#   s      r$   r   CosmosEmbedding.__init__H   s;    "=$]`a1-OMtM	r&   r'   timestepc                     U R                  U5      R                  U5      nU R                  U5      nU R                  U5      nXE4$ rS   )rb   type_asrc   rd   )r"   r'   rf   timesteps_projtembembedded_timesteps         r$   r9   CosmosEmbedding.forwardO   sA    199-H~. IIn5&&r&   )rd   rc   rb   )r;   r<   r=   r>   r?   r   rA   rB   
LongTensorr9   rC   rD   rE   s   @r$   rW   rW   G   sQ    Nc N# N$ N'U\\ 'U=M=M 'RWR^R^ ' 'r&   rW   c            	          ^  \ rS rSrS\S\SS4U 4S jjr SS\R                  S\R                  S	\\R                     S\R                  4S
 jjr	Sr
U =r$ )CosmosAdaLayerNormV   rI   hidden_featuresr   Nc                   > [         TU ]  5         Xl        [        R                  " 5       U l        [        R                  " USSS9U l        [        R                  " XSS9U l	        [        R                  " USU-  SS9U l
        g )NFr_   ra   r`   r   r
   )r   r   rY   r   rM   rN   	LayerNormrd   r    rL   rO   r"   rI   rq   r#   s      r$   r   CosmosAdaLayerNorm.__init__W   s_    ('')LLDQ			+UK		/1{?Or&   r'   rk   rj   c                 6   U R                  U5      nU R                  U5      nU R                  U5      nUb  X#SS SU R                  -  24   -   nUR	                  SSS9u  pEU R                  U5      nUR                  S:X  a  S XE4 5       u  pEUSU-   -  U-   nU$ )N.r
   dimc              3   B   #    U  H  oR                  S 5      v   M     g7fr   N	unsqueeze.0xs     r$   	<genexpr>-CosmosAdaLayerNorm.forward.<locals>.<genexpr>n   s     CNqKKNNN   r   )rN   rL   rO   rY   chunkrd   ndim)r"   r'   rk   rj   shiftscales         r$   r9   CosmosAdaLayerNorm.forward`   s     !OO,=> MM*;< MM*;< 1>VDDVDV@V>V9V4W W(..qb.9		-0!!Q&CUNCLE%U3e;r&   )rN   rY   rL   rO   rd   rS   )r;   r<   r=   r>   r?   r   rA   rB   r   r9   rC   rD   rE   s   @r$   ro   ro   V   sj    PC P# P$ P lp"\\>CllRZ[`[g[gRh	 r&   ro   c            	          ^  \ rS rSrSS\S\\   SS4U 4S jjjr SS\R                  S\R                  S	\\R                     S\R                  4S
 jjr	Sr
U =r$ )CosmosAdaLayerNormZerot   NrI   rq   r   c                 8  > [         TU ]  5         [        R                  " USSS9U l        [        R
                  " 5       U l        Uc  [        R                  " 5       U l        O[        R                  " XSS9U l        [        R                  " USU-  SS9U l
        g )NFr_   rs   r   r   )r   r   r   rt   rd   rM   rN   IdentityrL   r    rO   ru   s      r$   r   CosmosAdaLayerNormZero.__init__u   sl    LLDQ	'')"KKMDMIIkODM		/1{?Or&   r'   rk   rj   c                    U R                  U5      nU R                  U5      nU R                  U5      nUb  X#-   nUR                  SSS9u  pEnU R	                  U5      nUR
                  S:X  a  S XEU4 5       u  pEnUSU-   -  U-   nX4$ )Nr   rx   ry   r
   c              3   B   #    U  H  oR                  S 5      v   M     g7fr|   r}   r   s     r$   r   1CosmosAdaLayerNormZero.forward.<locals>.<genexpr>   s     !O:NQ++a..:Nr   r   )rN   rL   rO   r   rd   r   )r"   r'   rk   rj   r   r   gates          r$   r9   CosmosAdaLayerNormZero.forward   s     !OO,=> MM*;< MM*;< 1 8.44QB4?d		-0!!Q&!O5:N!OE$%U3e;""r&   )rN   rL   rO   rd   rS   )r;   r<   r=   r>   r?   r   r   rA   rB   r9   rC   rD   rE   s   @r$   r   r   t   sv    PC P(3- PSW P P" (,	#||# !<<# u||$	#
 
# #r&   r   c                       \ rS rSrS r   SS\S\R                  S\\R                     S\\R                     S\\R                     S	\R                  4S
 jjr	Sr
g)CosmosAttnProcessor2_0   c                 D    [        [        S5      (       d  [        S5      eg )Nscaled_dot_product_attentionzVCosmosAttnProcessor2_0 requires PyTorch 2.0. To use it, please upgrade PyTorch to 2.0.)hasattrFImportError)r"   s    r$   r   CosmosAttnProcessor2_0.__init__   s!    q899vww :r&   Nattnr'   encoder_hidden_statesattention_maskimage_rotary_embr   c           	         Uc  UnUR                  U5      nUR                  U5      nUR                  U5      nUR                  SUR                  S45      R                  SS5      nUR                  SUR                  S45      R                  SS5      nUR                  SUR                  S45      R                  SS5      nUR                  U5      nUR                  U5      nUb  SSKJ	n	  U	" XeSSS9nU	" XuSSS9n[        R                  R                  5       (       a  [        R                  " UR                  S5      UR                  S	9n
[        R                  " UR                  S5      UR                  S	9n[        R                  " UR                  S5      UR                  S	9nO3UR                  S5      n
UR                  S5      nUR                  S5      nUR!                  X-  SS
9nUR!                  X-  SS
9n["        R$                  " XgXSSS9nUR                  SS5      R'                  SS5      R)                  U5      nUR*                  S   " U5      nUR*                  S   " U5      nU$ )Nr
   rx   r   )apply_rotary_embT)use_realuse_real_unbind_dimr   )devicery   r\   F)	attn_mask	dropout_p	is_causalr   )to_qto_kto_v	unflattenheads	transposenorm_qnorm_k
embeddingsr   rA   onnxis_in_onnx_exporttensorsizer   repeat_interleaver   r   r0   rh   to_out)r"   r   r'   r   r   r   querykeyvaluer   	query_idxkey_idx	value_idxs                r$   __call__CosmosAttnProcessor2_0.__call__   s+    !($1!		-(ii-.		/0DJJ#34>>q!DmmA

B/0::1a@DJJ#34>>q!D E"kk# '5$UtacdE"34]_`C ::''))UZZ]5<<HIll388A;szzBGUZZ]5<<HI 

1IhhqkG

1I##I$8a#@''	(>A'F 663RW
 &//15==aCKKER A}5A}5r&    )NNN)r;   r<   r=   r>   r   r   rA   rB   r   r   rC   r   r&   r$   r   r      sz    x 9=153766 ||6  (5	6
 !.6 #5<<06 
6 6r&   r   c                   F  ^  \ rS rSr    SS\S\S\S\S\S\S\S	S
4U 4S jjjr    SS\	R                  S\	R                  S\	R                  S\\	R                     S\\	R                     S\\	R                     S\\	R                     S	\	R                  4S jjrSrU =r$ )CosmosTransformerBlock   num_attention_headsattention_head_dimcross_attention_dim	mlp_ratioadaln_lora_dimqk_normout_biasr   Nc                   > [         T	U ]  5         X-  n[        XS9U l        [	        US UUUSU[        5       S9U l        [        XS9U l        [	        UUUUUSU[        5       S9U l        [        XS9U l	        [        XSUS9U l        g )N)rI   rq   T)	query_dimr   r   dim_headr   ra   r   	processorgelu)multactivation_fnr   )r   r   r   norm1r   r   attn1norm2attn2norm3r   ff)
r"   r   r   r   r   r   r   r   hidden_sizer#   s
            r$   r   CosmosTransformerBlock.__init__   s     	)>+d
! $%'#,.	

 ,d
! 3%'#,.	

 ,d
kV^_r&   r'   r   rk   rj   r   extra_pos_embr   c                    Ub  X-   nU R                  XU5      u  pU R                  XS9n
XU
-  -   nU R                  XU5      u  pU R                  XUS9n
XU
-  -   nU R	                  XU5      u  pU R                  U5      nXU-  -   nU$ )N)r   )r   r   )r   r   r   r   r   r   )r"   r'   r   rk   rj   r   r   r   norm_hidden_statesr   attn_output	ff_outputs               r$   r9   CosmosTransformerBlock.forward  s     $)9M $(::mPT#U jj!3jW%{(:: $(::mPT#U jj\j ! 
 &{(:: $(::mPT#U GG./	%y(88r&   )r   r   r   r   r   r   )      @   rms_normF)NNNN)r;   r<   r=   r>   r?   floatstrr@   r   rA   rB   r   r9   rC   rD   rE   s   @r$   r   r      s    !!'` '`  '` !	'`
 '` '` '` '` 
'` '`\ (,370415||  %|| !<<	
 u||$ #5<<0  - !. 
 r&   r   c                      ^  \ rS rSr    SS\S\\\\4   S\\\\4   S\S\\\\4   SS4U 4S	 jjjrSS
\R                  S\
\   S\\R                  \R                  4   4S jjrSrU =r$ )CosmosRotaryPosEmbedi"  r   max_sizer   base_fps
rope_scaler   Nc                   > [         TU ]  5         [        X#5       VVs/ s H	  u  pgXg-  PM     snnU l        X0l        X@l        US-  S-  U l        US-  S-  U l        XR                  -
  U R                  -
  U l        US   U R                  U R                  S-
  -  -  U l	        US   U R                  U R                  S-
  -  -  U l
        US   U R                  U R                  S-
  -  -  U l        g s  snnf )Nr*   r
   r   r   )r   r   zipr   r   r   dim_hdim_wdim_th_ntk_factorw_ntk_factort_ntk_factor)	r"   r   r   r   r   r   r   patchr#   s	           r$   r   CosmosRotaryPosEmbed.__init__#  s     	:=h:ST:S;4:ST$  A%)
 A%)
 ::-

:
&qMdjjDJJN.KL&qMdjjDJJN.KL&qMdjjDJJN.KL Us   C3r'   fpsc                    UR                   u  p4pVnXPR                  S   -  X`R                  S   -  XpR                  S   -  /nUR                  n	SU R                  -  n
SU R                  -  nSU R
                  -  n[        R                  " [        U R                  5      U	[        R                  S9n[        R                  " SU R                  SU	[        R                  S9S U R                  S-   U R                  -  n[        R                  " SU R                  SU	[        R                  S9S U R                  S-   U R                  -  n[        R                  " SU R                  SU	[        R                  S9S U R                  S-   U R                  -  nSX-  -  nSX-  -  nSUU-  -  n[        R                  " US US    U5      S S S 2S S S 24   R                  US   SUS   S5      n[        R                  " US US    U5      S S S S 2S S 24   R                  US   US   SS5      nUc  [        R                  " US US    U5      nO-[        R                  " US US    U-  U R                   -  U5      nUS S 2S S S S 24   R                  SUS   US   S5      n[        R"                  " UUU/S-  SS9R%                  SS5      R'                  5       n[        R(                  " U5      n[        R*                  " U5      nUU4$ )	Nr   r   r
   g     @)r   dtype      ?rx   ry   )r-   r   r   r   r   r   rA   arangemaxr   float32r   r   r   outerrepeatr   catr0   r   cossin)r"   r'   r   r1   r2   r3   r4   r5   pe_sizer   h_thetaw_thetat_thetaseqdim_h_rangedim_w_rangedim_t_rangeh_spatial_freqsw_spatial_freqstemporal_freqsemb_hemb_wemb_tfreqsr   r  s                             r$   r9   CosmosRotaryPosEmbed.forward9  s   >K>Q>Q;
*e!33Vq?Q5QSX\k\klm\nSno%%D---D---D---ll3t}}-fEMMRLLDJJ&NObRVR\R\`aRacfjfpfpp 	 LLDJJ&NObRVR\R\`aRacfjfpfpp 	 LLDJJ&NObRVR\R\`aRacfjfpfpp 	 !56!56 45C'!*-?aq@PQXXY`abYcefhopqhrtuvC'!*-?dAq@PQXXY`abYcelmneoqrtuv ;KKLgaj 1>BE KKLgaj 1C 7$-- GXEatQ&'..q'!*gaj!L		5%/!3<DDQJPPRiiiiCxr&   )	r   r   r   r   r   r   r   r   r   )      r  r   r
   r
      g       @r   r   rS   )r;   r<   r=   r>   r?   r   r   r   rA   rB   r   r9   rC   rD   rE   s   @r$   r   r   "  s     *9+41@MM S#&M #sC-(	M
 M %-.M 
M M,&U\\ & &QVW\WcWcejeqeqWqQr & &r&   r   c                      ^  \ rS rSr SS\S\\\\4   S\\\\4   S\SS4
U 4S jjjrS	\R                  S\R                  4S
 jr
SrU =r$ )CosmosLearnablePositionalEmbedib  r   r   r   r`   r   Nc                   > [         TU ]  5         [        X#5       VVs/ s H	  u  pVXV-  PM     snnU l        X0l        X@l        [        R                  " [        R                  " U R                  S   U5      5      U l
        [        R                  " [        R                  " U R                  S   U5      5      U l        [        R                  " [        R                  " U R                  S   U5      5      U l        g s  snnf )Nr   r   r
   )r   r   r   r   r   r`   r   	ParameterrA   zeros	pos_emb_t	pos_emb_h	pos_emb_w)r"   r   r   r   r`   r   r   r#   s          r$   r   'CosmosLearnablePositionalEmbed.__init__c  s     	:=h:ST:S;4:ST$ekk$--2BK&PQekk$--2BK&PQekk$--2BK&PQ Us   C;r'   c           	      &   UR                   u  p#pEnX@R                  S   -  XPR                  S   -  X`R                  S   -  /nU R                  S US    S S S 2S S S S 24   R                  USUS   US   S5      nU R                  S US    S S S S 2S S S 24   R                  X'S   SUS   S5      n	U R
                  S US    S S S S S 2S S 24   R                  X'S   US   SS5      n
X-   U
-   nUR                  SS5      n[        R                  R                  USS[        R                  S9n[        R                  " U R                  U[        R                  " UR                  5       UR                  5       -  5      S9nX-  R!                  U5      $ )	Nr   r   r
   r   rx   T)rz   keepdimr   )alpha)r-   r   r  r   r  r  r0   rA   linalgvector_normr   addr`   npsqrtnumelrh   )r"   r'   r1   r2   r3   r4   r5   r  r  r  r  rT   rd   s                r$   r9   &CosmosLearnablePositionalEmbed.forwardt  s   >K>Q>Q;
*e!33Vq?Q5QSX\k\klm\nSno|,T1dD!-CDKKJXY[bcd[egnopgqstu|,T4D!-CDKKJ`aXbdegnopgqstu|,T4q!-CDKKJ`aXbdklmdnpqstume#kk!Q||''T'Wyy4rwwtzz|ciik7Q/RS
##M22r&   )r`   r   r   r  r  r  )r_   )r;   r<   r=   r>   r?   r   r   r   rA   rB   r9   rC   rD   rE   s   @r$   r  r  b  s     RR S#&R #sC-(	R
 R 
R R"3U\\ 3ell 3 3r&   r  c                     ^  \ rS rSrSrSr/ SQrS/rS/r\	             S!S\
S\
S	\
S
\
S\
S\S\
S\
S\\
\
\
4   S\\
\
\
4   S\\\\4   S\S\\   SS4U 4S jjj5       r     S"S\R$                  S\R$                  S\R$                  S\\R$                     S\\
   S\\R$                     S\\R$                     S\S\R$                  4S jjrS rU =r$ )#CosmosTransformer3DModeli  a  
A Transformer model for video-like data used in [Cosmos](https://github.com/NVIDIA/Cosmos).

Args:
    in_channels (`int`, defaults to `16`):
        The number of channels in the input.
    out_channels (`int`, defaults to `16`):
        The number of channels in the output.
    num_attention_heads (`int`, defaults to `32`):
        The number of heads to use for multi-head attention.
    attention_head_dim (`int`, defaults to `128`):
        The number of channels in each attention head.
    num_layers (`int`, defaults to `28`):
        The number of layers of transformer blocks to use.
    mlp_ratio (`float`, defaults to `4.0`):
        The ratio of the hidden layer size to the input size in the feedforward network.
    text_embed_dim (`int`, defaults to `4096`):
        Input dimension of text embeddings from the text encoder.
    adaln_lora_dim (`int`, defaults to `256`):
        The hidden dimension of the Adaptive LayerNorm LoRA layer.
    max_size (`Tuple[int, int, int]`, defaults to `(128, 240, 240)`):
        The maximum size of the input latent tensors in the temporal, height, and width dimensions.
    patch_size (`Tuple[int, int, int]`, defaults to `(1, 2, 2)`):
        The patch size to use for patchifying the input latent tensors in the temporal, height, and width
        dimensions.
    rope_scale (`Tuple[float, float, float]`, defaults to `(2.0, 1.0, 1.0)`):
        The scaling factor to use for RoPE in the temporal, height, and width dimensions.
    concat_padding_mask (`bool`, defaults to `True`):
        Whether to concatenate the padding mask to the input latent tensors.
    extra_pos_embed_type (`str`, *optional*, defaults to `learnable`):
        The type of extra positional embeddings to use. Can be one of `None` or `learnable`.
T)patch_embedfinal_layerrd   r   learnable_pos_embedr   r   r   r   
num_layersr   text_embed_dimr   r   r   r   concat_padding_maskextra_pos_embed_typer   Nc                   > [         TU ]  5         X4-  nU(       a  US-   OUn[        XU
SS9U l        [	        XIXS9U l        S U l        US:X  a  [        UU	U
S9U l        [        X5      U l	        [        R                  " [        U5       Vs/ s H  n[        UUUUUSSS9PM     sn5      U l        [        X5      U l        [        R"                  " XS	   U
S   -  U
S
   -  U-  SS9U l        SU l        g s  snf )Nr   Fr   )r   r   r   r   	learnable)r   r   r   r   )r   r   r   r   r   r   r   r   r
   )r   r   r   r-  r   roper/  r  rW   
time_embedr   
ModuleListranger   transformer_blocksro   norm_outr    proj_outgradient_checkpointing)r"   r   r   r   r   r0  r   r1  r   r   r   r   r2  r3  r   patch_embed_in_channels_r#   s                    r$   r   !CosmosTransformer3DModel.__init__  s'   " 	)> 6I+/k+,CR\chi )**
	 $( ;.'E'!%(D$ *+C #%-- z* +A '(;'9(6'#1&" +#
  +;G		AA6AFU\a
 ',#)s   C4r'   rf   r   r   r   condition_maskpadding_maskreturn_dictc	                 D  ^^^^ UR                   u  mppUb  [        R                  " X/SS9nU R                  R                  (       a  [
        R                  R                  U[        UR                   SS  5      [
        R                  R                  S9n[        R                  " XR                  S5      R                  TSU
SS5      /SS9nUb   UR                  S5      R                  S5      nU R                  XS9nU R                  R                  (       a  U R                  U5      OS nU R                  R                   u  nnnX-  mUU-  mUU-  mU R#                  U5      nUR%                  SS5      nUR&                  S:X  a  U R)                  X5      u  nnOtUR&                  S:X  ab  UR                   TSU
SS4:X  d   S	UR                    35       eUR%                  5       nU R)                  X5      u  nnUUUU4S
 jUU4 5       u  nnO eU R*                   HT  n[        R,                  " 5       (       a+  U R.                  (       a  U R1                  UUUUUUUU5      nMH  U" UUUUUUUS9nMV     U R3                  UUU5      nU R5                  U5      nUR7                  SUUUS45      nUR7                  STTT45      nUR9                  SSSSSSSS5      nUR%                  SS5      R%                  SS5      R%                  SS5      nU(       d  U4$ [;        US9$ )Nr   ry   r   )interpolationr
   )r   r   r+   z9Expected timestep to have shape [B, 1, T, 1, 1], but got c              3      >#    U  H=  nUR                  TTS S S5      R                  SSTTS5      R                  S S5      v   M?     g7f)r   rx   r   N)viewexpandr0   )r   r   r1   post_patch_heightpost_patch_num_framespost_patch_widths     r$   r   3CosmosTransformer3DModel.forward.<locals>.<genexpr>  sR      ' 3A z#8!QCB 13CRHA 3s   AA)r'   r   rk   rj   r   r   r   rx   r   r,   r*   r)   )sample)r-   rA   r   configr2  r   
functionalresizelistInterpolationModeNEARESTr~   r   r6  r3  r/  r   r-  r0   r   r7  r:  is_grad_enabledr=  _gradient_checkpointing_funcr;  r<  r   r/   r   )r"   r'   rf   r   r   r   rA  rB  rC  r2   r3   r4   r5   r   r   r6   r7   r8   rj   rk   blockr1   rI  rJ  rK  s                        @@@@r$   r9    CosmosTransformer3DModel.forward  s=    ?L>Q>Q;
Lf %!II}&E1MM;;**%0077d=#6#6rs#;<JLhLhLpLp 8 L "II 6 6q 9 @ @QPZ\]_` abhiM %+55a8BB1EN  99]9<CG;;CcCc00?im ..S# * 1"cM C<((7%--a3 ==A&*oom&N#D#]]a>>j!ZA%FF KHNNK[\F  '')H&*oom&N#D#'  12	'#D# 5 ,,E$$&&4+F+F $ A A!)%$!"	! !&"/*?&7%5"/#1! -0 m5FMm4%//Cc23FG%//4IK\^n3op &--aAq!Q1E%--a3;;AqAII!QO!##'}==r&   )r=  r/  r;  r-  r<  r6  r7  r:  )   rX      r     r   i   r   r  r  r  Tr5  )NNNNT)r;   r<   r=   r>   __doc__ _supports_gradient_checkpointing _skip_layerwise_casting_patterns_no_split_modules_keep_in_fp32_modulesr   r?   r   r   r@   r   r   r   rA   rB   r9   rC   rD   rE   s   @r$   r,  r,    s   B (,$'M$1223 #%"%"!)8+41@$(.9=,=, =, !	=,
  =, =, =, =, =, S#&=, #sC-(=, %-.=, "=, 'sm=, 
=, =,H 26!15/3 `>||`> ,,`>  %||	`>
 !.`> c]`> !.`> u||,`> `> 
`> `>r&   r,  )+typingr   r   numpyr'  rA   torch.nnr   torch.nn.functionalrO  r   configuration_utilsr   r   loadersr   utilsr	   	attentionr   attention_processorr   r   r   modeling_outputsr   modeling_utilsr   normalizationr   torchvisionr   Moduler   rG   rW   ro   r   r   r   r   r  r,  r   r&   r$   <module>rn     s    #      B - - # + " 7 ' # &ryy (bii 'bii ' <"#RYY "#J; ;|HRYY HV=299 =@3RYY 3BG>z;8N G>r&   