
    +hp              
          S SK r S SKJrJrJrJrJr  S SKrS SKJ	r	  S SK
J	s  Jr  SSKJrJr  SSKJrJr  SSKJrJrJrJrJr  SSKJr  SS	KJrJrJr  SS
KJ r   SSK!J"r"  SSK#J$r$J%r%J&r&J'r'  SSK(J)r)  SSK*J+r+  SSK,J-r-  \R\                  " \/5      r0SSS\Rb                  S\Rb                  4S jr2SSS\Rb                  4S jr3 " S S5      r4 " S S5      r5 " S S\R                  Rl                  \5      r7 " S S\R                  Rl                  5      r8 " S S\	Rl                  5      r9 " S  S!\	Rl                  5      r:\ " S" S#\	Rl                  5      5       r; " S$ S%\+\\\\"\5      r<g)&    N)AnyDictOptionalTupleUnion   )ConfigMixinregister_to_config)FromOriginalModelMixinPeftAdapterMixin)USE_PEFT_BACKEND	deprecateloggingscale_lora_layersunscale_lora_layers)maybe_allow_in_graph   )AttentionMixinAttentionModuleMixinFeedForward)dispatch_attention_fn)
CacheMixin)PixArtAlphaTextProjectionTimestepEmbedding	Timestepsget_1d_rotary_pos_embed)Transformer2DModelOutput)
ModelMixin)FP32LayerNormattnWanAttentionhidden_statesencoder_hidden_statesc                 b   Uc  UnU R                   (       ac  U R                  c#  U R                  U5      R                  SSS9u  p4nOfU R	                  U5      nU R                  U5      R                  SSS9u  pEO3U R	                  U5      nU R                  U5      nU R                  U5      nX4U4$ )Nr   dimr   )fused_projectionscross_attention_dim_headto_qkvchunkto_qto_kvto_kto_v)r    r"   r#   querykeyvalues         g/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/models/transformers/transformer_wan.py_get_qkv_projectionsr4   &   s    $ -((0 $M : @ @ @ KE IIm,E$9:@@@KJC		-(ii-.		/0u    encoder_hidden_states_imgc                     U R                   (       a$  U R                  U5      R                  SSS9u  p#X#4$ U R                  U5      nU R	                  U5      nX#4$ )Nr   r%   r&   )r(   to_added_kvr+   
add_k_proj
add_v_proj)r    r6   key_img	value_imgs       r3   _get_added_kv_projectionsr=   :   sd    !--.GHNNqVXNY  //";<OO$=>	r5   c                       \ rS rSrSrS r   SSSS\R                  S\\R                     S\\R                     S	\\	\R                  \R                  4      S
\R                  4S jjr
Srg)WanAttnProcessorC   Nc                 D    [        [        S5      (       d  [        S5      eg )Nscaled_dot_product_attentionzbWanAttnProcessor requires PyTorch 2.0. To use it, please upgrade PyTorch to version 2.0 or higher.)hasattrFImportErrorselfs    r3   __init__WanAttnProcessor.__init__F   s%    q899t  :r5   r    r!   r"   r#   attention_mask
rotary_embreturnc           
      .   S nUR                   b(  UR                  S   S-
  nUS S 2S U24   nUS S 2US 24   n[        XU5      u  pn
UR                  U5      nUR	                  U	5      n	UR                  SUR                  S45      nU	R                  SUR                  S45      n	U
R                  SUR                  S45      n
UbG  S[        R                  S[        R                  S[        R                  4S jnU" U/UQ76 nU" U	/UQ76 n	S nUb  [        X5      u  pUR                  U5      nUR                  SUR                  S45      nUR                  SUR                  S45      n[        UUUS S	S
U R                  S9nUR                  SS5      nUR                  U5      n[        UU	U
US	S
U R                  S9nUR                  SS5      nUR                  U5      nUb  X,-   nUR                  S   " U5      nUR                  S   " U5      nU$ )N   i   r   r%   r"   	freqs_cos	freqs_sinc                    U R                  SS5      R                  S5      u  p4USSS S24   nUSSS S24   n[        R                  " U 5      nX5-  XF-  -
  USSS S24'   X6-  XE-  -   USSS S24'   UR	                  U 5      $ )Nr%   )r%   r   .r   r   rN   )	unflattenunbindtorch
empty_liketype_as)r"   rO   rP   x1x2cossinouts           r3   apply_rotary_emb3WanAttnProcessor.__call__.<locals>.apply_rotary_embf   s    
 '00W=DDRHQTT	*QTT	*&&}5!#BH!4CAI!#BH!4CAI{{=11r5           F)	attn_mask	dropout_p	is_causalbackendr   r   )r9   shaper4   norm_qnorm_krR   headsrT   Tensorr=   norm_added_kr   _attention_backendflattenrV   to_out)rG   r    r"   r#   rJ   rK   r6   image_context_lengthr0   r1   r2   r\   hidden_states_imgr;   r<   s                  r3   __call__WanAttnProcessor.__call__L   sM    %)!??&#8#>#>q#AC#G (=aAVBVAV>V(W%$9!=Q=R:R$S!0F[\EE"kk#DJJ#34mmA

B/0DJJ#34!2$||2 <<2 !<<2 %U8Z8E"344C !$0!:4![G''0G''DJJ+;<G!++A

B/?@I 5//! !2 9 9!Q ? 1 9 9% @-$++
 &--a3%--e4()=MA}5A}5r5    NNN)__name__
__module____qualname____firstlineno__ri   rH   rT   rg   r   r   rn   __static_attributes__rp   r5   r3   r?   r?   C   s     9=15BFPP ||P  (5	P
 !.P U5<<#=>?P 
P Pr5   r?   c                       \ rS rSrS rSrg)WanAttnProcessor2_0   c                 4    Sn[        SSUSS9  [        U0 UD6$ )NzzThe WanAttnProcessor2_0 class is deprecated and will be removed in a future version. Please use WanAttnProcessor instead. rx   z1.0.0F)standard_warn)r   r?   )clsargskwargsdeprecation_messages       r3   __new__WanAttnProcessor2_0.__new__   s/    4 	 	'2EUZ[000r5   rp   N)rr   rs   rt   ru   r   rv   rp   r5   r3   rx   rx      s    1r5   rx   c                   b  ^  \ rS rSr\r\/r        SS\S\S\S\S\S\	\   S\	\   4U 4S	 jjjr
S
 r\R                  " 5       S 5       r   SS\R                  S\	\R                     S\	\R                     S\	\\R                  \R                  4      S\R                  4
S jjrSrU =r$ )r!      r'   rf   dim_headepsdropoutadded_kv_proj_dimr)   c
                 V  > [         T
U ]  5         X2-  U l        X l        X`l        Xpl        Uc  U R                  OXr-  U l        [        R                  R                  XR                  SS9U l
        [        R                  R                  XR                  SS9U l        [        R                  R                  XR                  SS9U l        [        R                  R                  [        R                  R                  U R                  USS9[        R                  R                  U5      /5      U l        [        R                  R!                  X2-  USS9U l        [        R                  R!                  X2-  USS9U l        S =U l        U l        Ub  [        R                  R                  X`R                  SS9U l        [        R                  R                  X`R                  SS9U l        [        R                  R!                  X2-  US9U l        US LU l        U R/                  U5        g )NTbias)r   elementwise_affine)r   )superrH   	inner_dimrf   r   r)   kv_inner_dimrT   nnLinearr,   r.   r/   
ModuleListDropoutrk   RMSNormrd   re   r9   r:   rh   is_cross_attentionset_processor)rG   r'   rf   r   r   r   r   r)   	processorr   	__class__s             r3   rH   WanAttention.__init__   s    	!)
!2(@%.F.NDNNTlTtHHOOCdOC	HHOOC):):OF	HHOOC):):OF	hh))$?  )
 hh&&x'7SUY&Zhh&&x'7SUY&Z,00$/(#hhoo.?VZo[DO#hhoo.?VZo[DO % 0 01As 0 KD":$"F9%r5   c                    [        U SS5      (       a  g U R                  GcL  [        R                  " U R                  R
                  R                  U R                  R
                  R                  U R                  R
                  R                  /5      n[        R                  " U R                  R                  R                  U R                  R                  R                  U R                  R                  R                  /5      nUR                  u  p4[        R                  " S5         [        R                  " XCSS9U l        S S S 5        U R                  R                  XS.SSS9  GO[        R                  " U R                  R
                  R                  U R                  R
                  R                  /5      n[        R                  " U R                  R                  R                  U R                  R                  R                  /5      nUR                  u  p4[        R                  " S5         [        R                  " XCSS9U l        S S S 5        U R                   R                  XS.SSS9  U R"                  Gb  [        R                  " U R$                  R
                  R                  U R&                  R
                  R                  /5      n[        R                  " U R$                  R                  R                  U R&                  R                  R                  /5      nUR                  u  p4[        R                  " S5         [        R                  " XCSS9U l        S S S 5        U R(                  R                  XS.SSS9  SU l        g ! , (       d  f       GN\= f! , (       d  f       GN`= f! , (       d  f       NW= f)Nr(   FmetaTr   )weightr   )strictassign)getattrr)   rT   catr,   r   datar.   r/   r   rc   devicer   r   r*   load_state_dictr-   r   r9   r:   r8   r(   )rG   concatenated_weightsconcatenated_biasout_featuresin_featuress        r3   fuse_projectionsWanAttention.fuse_projections   s   4,e44((0#(99dii.>.>.C.CTYYEUEUEZEZ\`\e\e\l\l\q\q-r#s  %		499>>+>+>		@S@SUYU^U^UcUcUhUh*i j(<(B(B%Lf% iiM &KK''/KTXae (  $)99dii.>.>.C.CTYYEUEUEZEZ-[#\  %		499>>+>+>		@S@S*T U(<(B(B%Lf%YY{tL
 &JJ&&/KTXae '  !!-#(99doo.D.D.I.I4??KaKaKfKf-g#h  %		4??+?+?+D+DdooFZFZF_F_*` a(<(B(B%Lf%#%99[T#R  &,,/KTXae -  "&3 &% &% &%s$   ,N:N.O 
N+.
N= 
Oc                     [        U SS5      (       d  g [        U S5      (       a  [        U S5        [        U S5      (       a  [        U S5        [        U S5      (       a  [        U S5        SU l        g )Nr(   Fr*   r-   r8   )r   rC   delattrr(   rF   s    r3   unfuse_projectionsWanAttention.unfuse_projections   sb    t0%884""D(#4!!D'"4''D-(!&r5   r"   r#   rJ   rK   rL   c                 ,    U R                   " XX#U40 UD6$ N)r   )rG   r"   r#   rJ   rK   r~   s         r3   forwardWanAttention.forward  s     ~~d3HZdohnoor5   )r9   r:   r   r)   r(   rf   r   r   r   rh   re   rd   r8   r.   r-   rk   r,   r*   r/   )   @   gh㈵>r^   NNNNrq   )rr   rs   rt   ru   r?   _default_processor_cls_available_processorsintfloatr   rH   r   rT   no_gradr   rg   r   r   rv   __classcell__r   s   @r3   r!   r!      s    --.
 +/26(&(& (& 	(&
 (& (& $C=(& #+3-(& (&T!&F ]]_' '  9=15BFp||p  (5p !.	p
 U5<<#=>?p 
p pr5   c                   r   ^  \ rS rSrS	S\S\4U 4S jjjrS\R                  S\R                  4S jrSr	U =r
$ )
WanImageEmbeddingi  r   r   c                    > [         TU ]  5         [        U5      U l        [	        XSSS9U l        [        U5      U l        Ub1  [        R                  " [        R                  " SX15      5      U l        g S U l        g )NrN   gelu)multactivation_fn)r   rH   r   norm1r   ffnorm2r   	ParameterrT   zeros	pos_embed)rG   r   r   pos_embed_seq_lenr   s       r3   rH   WanImageEmbedding.__init__  s_    ";/
kavV"<0
(\\%++a9J*XYDN!DNr5   encoder_hidden_states_imagerL   c                     U R                   b3  UR                  u  p#nUR                  SSU-  U5      nXR                   -   nU R                  U5      nU R	                  U5      nU R                  U5      nU$ )Nr%   r   )r   rc   viewr   r   r   )rG   r   
batch_sizeseq_len	embed_dimr"   s         r3   r   WanImageEmbedding.forward  su    >>%-H-N-N*J*E*J*J2qSZ{\e*f'*E*V'

#>?.

=1r5   )r   r   r   r   r   )rr   rs   rt   ru   r   rH   rT   rg   r   rv   r   r   s   @r3   r   r     s;    	"C 	"s 	" 	"	5<< 	ELL 	 	r5   r   c                      ^  \ rS rSr  SS\S\S\S\S\\   S\\   4U 4S jjjr  SS	\R                  S
\R                  S\\R                     S\\   4S jjr	Sr
U =r$ )WanTimeTextImageEmbeddingi+  r'   time_freq_dimtime_proj_dimtext_embed_dimimage_embed_dimr   c                   > [         TU ]  5         [        USSS9U l        [	        X!S9U l        [        R                  " 5       U l        [        R                  " X5      U l
        [        XASS9U l        S U l        Ub  [        XQUS9U l        g g )NTr   )num_channelsflip_sin_to_cosdownscale_freq_shift)in_channelstime_embed_dim	gelu_tanh)act_fn)r   )r   rH   r   timesteps_projr   time_embedderr   SiLUr   r   	time_projr   text_embedderimage_embedderr   )rG   r'   r   r   r   r   r   r   s          r3   rH   "WanTimeTextImageEmbedding.__init__,  s     	']TXopq.=]ggi366~S^_"&"3O\m"nD 'r5   timestepr#   r   timestep_seq_lenc                    U R                  U5      nUb  UR                  SSU45      n[        [        U R                  R                  5       5      5      R                  nUR                  U:w  a%  U[        R                  :w  a  UR                  U5      nU R	                  U5      R                  U5      nU R                  U R                  U5      5      nU R                  U5      nUb  U R                  U5      nXgX#4$ )Nr   r%   )r   rR   nextiterr   
parametersdtyperT   int8torV   r   r   r   r   )rG   r   r#   r   r   time_embedder_dtypetembtimestep_projs           r3   r   !WanTimeTextImageEmbedding.forwardA  s     &&x0'))!b2B-CDH"4(:(:(E(E(G#HIOO>>005HEJJ5V{{#67H!!(+334IJt{{4'89 $ 2 23H I&2*.*=*=>Y*Z'$9VVr5   )r   r   r   r   r   r   )NN)rr   rs   rt   ru   r   r   rH   rT   rg   r   rv   r   r   s   @r3   r   r   +  s     *.+/oo o 	o
 o "#o $C=o o2 ?C*.W,,W  %||W &.ell%;	W
 #3-W Wr5   r   c            	          ^  \ rS rSr SS\S\\\\4   S\S\4U 4S jjjrS\R                  S\R                  4S	 jr
S
rU =r$ )WanRotaryPosEmbediY  attention_head_dim
patch_sizemax_seq_lenthetac           
        > [         TU ]  5         Xl        X l        X0l        SUS-  -  =pVX-
  U-
  n[
        R                  R                  R                  5       (       a  [
        R                  O[
        R                  n/ n	/ n
XuU4 H5  n[        UUUSSUS9u  pU	R                  U5        U
R                  U5        M7     U R                  S[
        R                  " U	SS9SS	9  U R                  S
[
        R                  " U
SS9SS	9  g )Nr      T)use_realrepeat_interleave_realfreqs_dtyperO   rN   r&   F)
persistentrP   )r   rH   r   r   r   rT   backendsmpsis_availablefloat32float64r   appendregister_bufferr   )rG   r   r   r   r   h_dimw_dimt_dimr   rO   rP   r'   freq_cosfreq_sinr   s                 r3   rH   WanRotaryPosEmbed.__init__Z  s    	"4$&/1455"*U2',~~'9'9'F'F'H'Hemmemm		%(C!8'+'"H X&X& ) 	[%))I1*ERWX[%))I1*ERWXr5   r"   rL   c                    UR                   u  p#pEnU R                  u  pxn	XG-  XX-  Xi-  pn
U R                  SU R                  S-  -  -
  U R                  S-  U R                  S-  /nU R                  R	                  USS9nU R
                  R	                  USS9nUS   S U
 R                  U
SSS5      R                  XUS5      nUS   S U R                  SUSS5      R                  XUS5      nUS   S U R                  SSUS5      R                  XUS5      nUS   S U
 R                  U
SSS5      R                  XUS5      nUS   S U R                  SUSS5      R                  XUS5      nUS   S U R                  SSUS5      R                  XUS5      n[        R                  " UUU/SS9R                  SX-  U-  SS5      n[        R                  " UUU/SS9R                  SX-  U-  SS5      nX4$ )Nr   r   rN   r&   r   r%   )rc   r   r   rO   splitrP   r   expandrT   r   reshape)rG   r"   r   r   
num_framesheightwidthp_tp_hp_wppfpphppwsplit_sizesrO   rP   freqs_cos_ffreqs_cos_hfreqs_cos_wfreqs_sin_ffreqs_sin_hfreqs_sin_ws                         r3   r   WanRotaryPosEmbed.forward}  sC   >K>Q>Q;
*e#")6=%,# ##a4+B+Ba+G&HH##q(##q(
 NN((!(<	NN((!(<	l4C(--c1a<CCCcSUVl4C(--aa<CCCcSUVl4C(--aC<CCCcSUVl4C(--c1a<CCCcSUVl4C(--aa<CCCcSUVl4C(--aC<CCCcSUVII{KE2NVVWXZ]ZcfiZiklnpq	II{KE2NVVWXZ]ZcfiZiklnpq	##r5   )r   r   r   )g     @)rr   rs   rt   ru   r   r   r   rH   rT   rg   r   rv   r   r   s   @r3   r   r   Y  sj     !Y!Y #sC-(!Y 	!Y
 !Y !YF$U\\ $ell $ $r5   r   c                      ^  \ rS rSr    SS\S\S\S\S\S\S\\   4U 4S	 jjjr	S
\
R                  S\
R                  S\
R                  S\
R                  S\
R                  4
S jrSrU =r$ )WanTransformerBlocki  r'   ffn_dim	num_headsqk_normcross_attn_normr   r   c           
        > [         TU ]  5         [        XSS9U l        [	        UUX-  US [        5       S9U l        [	        UUX-  UUX-  [        5       S9U l        U(       a
  [        XSS9O[        R                  " 5       U l
        [        XSS9U l        [        XSS9U l        [        R                  " [        R                   " SS	U5      US
-  -  5      U l        g )NFr   )r'   rf   r   r   r)   r   )r'   rf   r   r   r   r)   r   Tzgelu-approximate)r   r   rN   r         ?)r   rH   r   r   r!   r?   attn1attn2r   Identityr   r   ffnnorm3r   rT   randnscale_shift_table)	rG   r'   r!  r"  r#  r$  r   r   r   s	           r3   rH   WanTransformerBlock.__init__  s     	 #3F
!%%)&(

 "%/%(%5&(

 JY]3E^`^i^i^k
 sEWX"3F
!#ekk!Q.DsCx.O!Pr5   r"   r#   r   rK   rL   c                 ,   UR                   S:X  a  U R                  R                  S5      UR                  5       -   R	                  SSS9u  pVpxpUR                  S5      nUR                  S5      nUR                  S5      nUR                  S5      nU	R                  S5      n	U
R                  S5      n
O/U R                  UR                  5       -   R	                  SSS9u  pVpxpU R                  UR                  5       5      SU-   -  U-   R                  U5      nU R                  US S U5      nUR                  5       X-  -   R                  U5      nU R                  UR                  5       5      R                  U5      nU R                  XS S 5      nX-   nU R                  UR                  5       5      SU	-   -  U-   R                  U5      nU R                  U5      nUR                  5       UR                  5       U
-  -   R                  U5      nU$ )N   r   r   r   r&   rN   )ndimr.  	unsqueezer   r+   squeezer   rV   r(  r   r)  r,  r+  )rG   r"   r#   r   rK   	shift_msa	scale_msagate_msac_shift_msac_scale_msa
c_gate_msanorm_hidden_statesattn_output	ff_outputs                 r3   r   WanTransformerBlock.forward  s     99> &&003djjlBeA1eo QI( "))!,I!))!,I''*H%--a0K%--a0K#++A.J &&5eA1eo QI(
 #jj)<)<)>?1y=QT]]ffgtujj!3T4L&,,.1GGPPQ^_ "ZZ(;(;(=>FF}Ujj!3DRVW%3 #jj)<)<)>?1{?SVaajj
 HH/0	&,,.1BZ1OOXXYfgr5   )r(  r)  r+  r   r   r,  r.  )rms_norm_across_headsFư>N)rr   rs   rt   ru   r   strboolr   r   rH   rT   rg   r   rv   r   r   s   @r3   r   r     s     / %+/'Q'Q 'Q 	'Q
 'Q 'Q 'Q $C='Q 'QR*||*  %||* ll	*
 LL* 
* *r5   r   c            $         ^  \ rS rSrSrSr/ SQrS/r/ SQrS/r	S/r
\                S#S	\\   S
\S\S\S\S\S\S\S\S\S\\   S\S\\   S\\   S\S\\   SS4"U 4S jjj5       r   S$S\R(                  S\R*                  S\R(                  S\\R(                     S\S \\\\4      S\\R(                  \\\R(                  4   4   4S! jjrS"rU =r$ )%WanTransformer3DModeli  a<  
A Transformer model for video-like data used in the Wan model.

Args:
    patch_size (`Tuple[int]`, defaults to `(1, 2, 2)`):
        3D patch dimensions for video embedding (t_patch, h_patch, w_patch).
    num_attention_heads (`int`, defaults to `40`):
        Fixed length for text embeddings.
    attention_head_dim (`int`, defaults to `128`):
        The number of channels in each head.
    in_channels (`int`, defaults to `16`):
        The number of channels in the input.
    out_channels (`int`, defaults to `16`):
        The number of channels in the output.
    text_dim (`int`, defaults to `512`):
        Input dimension for text embeddings.
    freq_dim (`int`, defaults to `256`):
        Dimension for sinusoidal time embeddings.
    ffn_dim (`int`, defaults to `13824`):
        Intermediate dimension in feed-forward network.
    num_layers (`int`, defaults to `40`):
        The number of layers of transformer blocks to use.
    window_size (`Tuple[int]`, defaults to `(-1, -1)`):
        Window size for local attention (-1 indicates global attention).
    cross_attn_norm (`bool`, defaults to `True`):
        Enable cross-attention normalization.
    qk_norm (`bool`, defaults to `True`):
        Enable query/key normalization.
    eps (`float`, defaults to `1e-6`):
        Epsilon value for normalization layers.
    add_img_emb (`bool`, defaults to `False`):
        Whether to use img_emb.
    added_kv_proj_dim (`int`, *optional*, defaults to `None`):
        The number of channels to use for the added key and value projections. If `None`, no projection is used.
T)patch_embeddingcondition_embeddernormr   )r   r.  r   r   r,  norm_added_qNr   num_attention_headsr   r   out_channelstext_dimfreq_dimr!  
num_layersr$  r#  r   	image_dimr   rope_max_seq_lenr   rL   c                 R  > [         TU ]  5         X#-  nU=(       d    Un[        X1U5      U l        [        R
                  " UUXS9U l        [        UUUS-  UUUS9U l        [        R                  " [        U	5       Vs/ s H  n[        UXXX5      PM     sn5      U l        [        UUSS9U l        [        R                  " UU[         R"                  " U5      -  5      U l        [        R&                  " [(        R*                  " SSU5      US-  -  5      U l        SU l        g s  snf )	N)kernel_sizestrider   )r'   r   r   r   r   r   Fr&  rN   r   r'  )r   rH   r   roper   Conv3drE  r   rF  r   ranger   blocksr   norm_outr   mathprodproj_outr   rT   r-  r.  gradient_checkpointing)rG   r   rI  r   r   rJ  rK  rL  r!  rM  r$  r#  r   rN  r   rO  r   r   _r   s                      r3   rH   WanTransformer3DModel.__init__  s   ( 	'<	#2{ &&8FVW	!yyiZk #<"#a-#%/#
 mm
 z*	 +A $wWWZ +	
 &iO		)\DIIj<Q-QR!#ekk!Q	.JYX[^.[!\&+#s   >D$r"   r   r#   r   return_dictattention_kwargsc           
      j   Ub#  UR                  5       nUR                  SS5      nOSn[        (       a  [        X5        O+Ub(  UR	                  SS 5      b  [
        R                  S5        UR                  u  ppnU R                  R                  u  pnX-  nX-  nX-  nU R                  U5      nU R                  U5      nUR                  S5      R                  SS5      nUR                  S:X  a   UR                  S   nUR                  5       nOS nU R                  X#UUS9u  nnp4Ub  UR!                  SS5      nOUR!                  SS5      nUb  ["        R$                  " XC/SS9n["        R&                  " 5       (       a9  U R(                  (       a(  U R*                   H  nU R-                  UXUU5      nM     OU R*                   H  nU" XUU5      nM     UR                  S	:X  az  U R.                  R1                  S
5      R3                  UR4                  5      UR1                  S5      -   R7                  SSS9u  nnUR9                  S5      nUR9                  S5      nOHU R.                  R3                  UR4                  5      UR1                  S5      -   R7                  SSS9u  nnUR3                  UR4                  5      nUR3                  UR4                  5      nU R;                  UR=                  5       5      SU-   -  U-   R?                  U5      nU RA                  U5      nURC                  UUUUXUS5      nURE                  S
SSSSSS	S5      nUR                  SS5      R                  SS5      R                  SS	5      n[        (       a  [G        X5        U(       d  U4$ [I        US9$ )Nscaleg      ?zVPassing `scale` via `attention_kwargs` when not using the PEFT backend is ineffective.r   rN   )r   )r   r%   r&   r   r   r%      r1     r   )sample)%copypopr   r   getloggerwarningrc   configr   rS  rE  rj   	transposer2  rF  rR   rT   concatis_grad_enabledr[  rV  _gradient_checkpointing_funcr.  r3  r   r   r+   r4  rW  r   rV   rZ  r  permuter   r   )rG   r"   r   r#   r   r^  r_  
lora_scaler   r   r  r  r  r  r  r  post_patch_num_framespost_patch_heightpost_patch_widthrK   
ts_seq_lenr   r   blockshiftra  outputs                              r3   r   WanTransformer3DModel.forwardX  s    '/446)--gs;JJd/+0@0D0DWd0S0_l ?L>Q>Q;
*e..# * 1"M <YY}-
,,];%--a0::1a@ ==A!*J'')HJRVRiRi-H[e Sj S
Om2 !)33Aw?M *33Aw?M&2$)LL2M1ekl$m!   ""t'B'B $ A A=PZ! %
  %mM[e f % 99> 22<<Q?BB4;;ORVR`R`abRccjjklrsjtLE5MM!$EMM!$E !2255dkkBT^^TUEVV]]^_ef]gLE5 --.--.}':':'<=UKeS\\]jkm4%---/@BRTW^ace
 &--aAq!Q1E&&q!,44Q:BB1aH19'v66r5   )rV  rF  r[  rW  rE  rZ  rS  r.  ))rN   r   r   (         r{  i      i 6  ry  Tr?  r@  NNi   N)NTN)rr   rs   rt   ru   __doc__ _supports_gradient_checkpointing _skip_layerwise_casting_patterns_no_split_modules_keep_in_fp32_modules"_keys_to_ignore_on_load_unexpected_repeated_blocksr
   r   r   rB  r   rA  r   rH   rT   rg   
LongTensorr   r   r   r   rv   r   r   s   @r3   rD  rD    s   "H (,$'X$./]*8)9&-. "+#%"% $!8#'+/ $+/#6,#J6, !6,  	6,
 6, 6, 6, 6, 6, 6, 6, #6, 6, C=6, $C=6,  !6," $C=#6,$ 
%6, 6,z ?C 59b7||b7 ""b7  %||	b7
 &.ell%;b7 b7 #4S>2b7 
u||T#u||"344	5b7 b7r5   rD  )=rX  typingr   r   r   r   r   rT   torch.nnr   torch.nn.functional
functionalrD   configuration_utilsr	   r
   loadersr   r   utilsr   r   r   r   r   utils.torch_utilsr   	attentionr   r   r   attention_dispatchr   cache_utilsr   
embeddingsr   r   r   r   modeling_outputsr   modeling_utilsr   normalizationr   
get_loggerrr   rh  rg   r4   r=   r?   rx   Moduler!   r   r   r   r   rD  rp   r5   r3   <module>r     s>    4 4     B ? a a 5 I I 6 $ i i 7 ' ) 
		H	%~ ell chcoco (N u|| Y Yx1 1gp588??$8 gpT 0+W		 +W\=$		 =$@ T")) T TnI7-/EzSaI7r5   