
    +h                     L   S SK JrJrJrJr  S SKrS SKJr  S SKJs  J	r
  S SKrSSKJrJr  SSKJr  SSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJrJr  \R<                  " \5      r Sr! " S S\RD                  5      r# " S S\RD                  5      r$ " S S\RJ                  5      r& " S S\RD                  5      r' " S S\RP                  5      r) " S S\RD                  5      r* " S S\RD                  5      r+ " S S\RD                  5      r, " S S\RD                  5      r- " S  S!\RD                  5      r. " S" S#\RD                  5      r/ " S$ S%\RD                  5      r0 " S& S'\RD                  5      r1 " S( S)\RD                  5      r2S* r3S+ r4 " S, S-\\\5      r5g).    )ListOptionalTupleUnionN   )ConfigMixinregister_to_config)FromOriginalModelMixin)logging)apply_forward_hook   )get_activation)AutoencoderKLOutput)
ModelMixin   )DecoderOutputDiagonalGaussianDistributionc                   h   ^  \ rS rSr SU 4S jjrS\R                  S\R                  4S jrSrU =r	$ )	AvgDown3D%   c                   > [         TU ]  5         Xl        X l        X0l        X@l        U R                  U R
                  -  U R
                  -  U l        XR                  -  U-  S:X  d   eXR                  -  U-  U l        g Nr   )super__init__in_channelsout_channelsfactor_tfactor_sfactor
group_sizeselfr   r   r   r   	__class__s        j/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/models/autoencoders/autoencoder_kl_wan.pyr   AvgDown3D.__init__&   sp     	&(  mmdmm3dmmC[[(<71<<<%3|C    xreturnc           
         U R                   UR                  S   U R                   -  -
  U R                   -  nSSSSUS4n[        R                  " X5      nUR                  u  pEpgnUR	                  UUX`R                   -  U R                   XpR
                  -  U R
                  XR
                  -  U R
                  5      nUR                  SSSSSSSS5      R                  5       nUR	                  UXPR                  -  X`R                   -  XpR
                  -  XR
                  -  5      nUR	                  UU R                  U R                  X`R                   -  XpR
                  -  XR
                  -  5      nUR                  SS	9nU$ )
Nr   r   r   r               dim)r   shapeFpadviewr   permute
contiguousr   r   r    mean)	r"   r'   pad_tr2   BCTHWs	            r$   forwardAvgDown3D.forward7   sU   dmm!;;t}}L!Q5!$EE!MaAFFMMMMMM	
 IIaAq!Q1-88:FFO
 FFOO
 FFqFMr&   )r   r   r   r    r   r   r   )
__name__
__module____qualname____firstlineno__r   torchTensorr=   __static_attributes____classcell__r#   s   @r$   r   r   %   s.     D"   %,,    r&   r   c                   x   ^  \ rS rSr S	S\S\4U 4S jjjrS
S\R                  S\R                  4S jjrSr	U =r
$ )DupUp3DZ   r   r   c                   > [         TU ]  5         Xl        X l        X0l        X@l        U R                  U R
                  -  U R
                  -  U l        X R                  -  U-  S:X  d   eX R                  -  U-  U l        g r   )r   r   r   r   r   r   r   repeatsr!   s        r$   r   DupUp3D.__init__[   sp     	&(  mmdmm3dmmCkk)K71<<<#kk1[@r&   r'   r(   c                    UR                  U R                  SS9nUR                  UR                  S5      U R                  U R
                  U R                  U R                  UR                  S5      UR                  S5      UR                  S5      5      nUR                  SSSSSSS	S5      R                  5       nUR                  UR                  S5      U R                  UR                  S5      U R
                  -  UR                  S5      U R                  -  UR                  S5      U R                  -  5      nU(       a!  US S 2S S 2U R
                  S-
  S 2S S 2S S 24   nU$ )
Nr   r.   r   r   r   r,   r*   r-   r+   )	repeat_interleaverM   r3   sizer   r   r   r4   r5   )r"   r'   first_chunks      r$   r=   DupUp3D.forwardm   s#   !4FFFF1IMMMMMMFF1IFF1IFF1I	
 IIaAq!Q1-88:FFFF1IFF1I%FF1I%FF1I%
 !Q)+Q12Ar&   )r   r   r   r   r   rM   r?   )F)r@   rA   rB   rC   intr   rD   rE   r=   rF   rG   rH   s   @r$   rJ   rJ   Z   sH     AA A A$ U\\  r&   rJ   c                      ^  \ rS rSrSr  SS\S\S\\\\\\4   4   S\\\\\\4   4   S\\\\\\4   4   SS	4U 4S
 jjjrSU 4S jjr	Sr
U =r$ )WanCausalConv3d   aM  
A custom 3D causal convolution layer with feature caching support.

This layer extends the standard Conv3D layer by ensuring causality in the time dimension and handling feature
caching for efficient inference.

Args:
    in_channels (int): Number of channels in the input image
    out_channels (int): Number of channels produced by the convolution
    kernel_size (int or tuple): Size of the convolving kernel
    stride (int or tuple, optional): Stride of the convolution. Default: 1
    padding (int or tuple, optional): Zero-padding added to all three sides of the input. Default: 0
r   r   kernel_sizestridepaddingr(   Nc                    > [         TU ]  UUUUUS9  U R                  S   U R                  S   U R                  S   U R                  S   SU R                  S   -  S4U l        SU l        g )N)r   r   rX   rY   rZ   r   r   r   r   r   r   )r   r   rZ   _padding)r"   r   r   rX   rY   rZ   r#   s         r$   r   WanCausalConv3d.__init__   sx     	#%# 	 	
 a$,,q/4<<?DLLYZO]^aeamamnoap]prst r&   c                 :  > [        U R                  5      nUb^  U R                  S   S:  aK  UR                  UR                  5      n[        R
                  " X!/SS9nUS==   UR                  S   -  ss'   [        R                  " X5      n[        TU ])  U5      $ )Nr,   r   r   r.   )listr]   todevicerD   catr0   r1   r2   r   r=   )r"   r'   cache_xrZ   r#   s       r$   r=   WanCausalConv3d.forward   s    t}}%4==#3a#7jj*G		7,A.AAJ'--**JEE!wq!!r&   )r]   rZ   )r   r   N)r@   rA   rB   rC   __doc__rT   r   r   r   r=   rF   rG   rH   s   @r$   rV   rV      s    & 4545!! ! 3c3m 445	!
 c5c3//0! sE#sC-001! 
! !(" "r&   rV   c                   N   ^  \ rS rSrSrSS\S\S\S\SS4
U 4S	 jjjrS
 rSr	U =r
$ )WanRMS_norm   a  
A custom RMS normalization layer.

Args:
    dim (int): The number of dimensions to normalize over.
    channel_first (bool, optional): Whether the input tensor has channels as the first dimension.
        Default is True.
    images (bool, optional): Whether the input represents image data. Default is True.
    bias (bool, optional): Whether to include a learnable bias term. Default is False.
r/   channel_firstimagesbiasr(   Nc                 R  > [         TU ]  5         U(       d  SOSnU(       a  U/UQ7OU4nX l        US-  U l        [        R
                  " [        R                  " U5      5      U l        U(       a0  [        R
                  " [        R                  " U5      5      U l
        g SU l
        g )N)r   r   r   )r   r   g      ?        )r   r   rk   scalenn	ParameterrD   onesgammazerosrm   )r"   r/   rk   rl   rm   broadcastable_dimsr0   r#   s          r$   r   WanRMS_norm.__init__   sw    .4Y&.;*)*#*#X
\\%**U"34
8<BLLU!34	#	r&   c                     [         R                  " XR                  (       a  SOSS9U R                  -  U R                  -  U R
                  -   $ )Nr   r.   )r1   	normalizerk   rp   rt   rm   )r"   r'   s     r$   r=   WanRMS_norm.forward   s<    {{1(:(:1DtzzQTXT^T^^aeajajjjr&   )rm   rk   rt   rp   TTF)r@   rA   rB   rC   rg   rT   boolr   r=   rF   rG   rH   s   @r$   ri   ri      sJ    	FC F FT FX\ Fim F Fk kr&   ri   c                   ,   ^  \ rS rSrSrU 4S jrSrU =r$ )WanUpsample   z
Perform upsampling while ensuring the output tensor has the same data type as the input.

Args:
    x (torch.Tensor): Input tensor to be upsampled.

Returns:
    torch.Tensor: Upsampled tensor with the same data type as the input.
c                 \   > [         TU ]  UR                  5       5      R                  U5      $ rf   )r   r=   floattype_as)r"   r'   r#   s     r$   r=   WanUpsample.forward   s#    wqwwy)11!44r&    )r@   rA   rB   rC   rg   r=   rF   rG   rH   s   @r$   r   r      s    5 5r&   r   c            	       T   ^  \ rS rSrSrSS\S\S\SS4U 4S jjjrSS	/4S
 jrSr	U =r
$ )WanResample   aP  
A custom resampling module for 2D and 3D data.

Args:
    dim (int): The number of input/output channels.
    mode (str): The resampling mode. Must be one of:
        - 'none': No resampling (identity operation).
        - 'upsample2d': 2D upsampling with nearest-exact interpolation and convolution.
        - 'upsample3d': 3D upsampling with nearest-exact interpolation, convolution, and causal 3D convolution.
        - 'downsample2d': 2D downsampling with zero-padding and convolution.
        - 'downsample3d': 3D downsampling with zero-padding, convolution, and causal 3D convolution.
Nr/   modeupsample_out_dimr(   c           
        > [         TU ]  5         Xl        X l        Uc  US-  nUS:X  a9  [        R
                  " [        SSS9[        R                  " XSSS95      U l        g US	:X  aL  [        R
                  " [        SSS9[        R                  " XSSS95      U l        [        XS-  S
SS9U l
        g US:X  aE  [        R
                  " [        R                  " S5      [        R                  " XSSS95      U l        g US:X  aV  [        R
                  " [        R                  " S5      [        R                  " XSSS95      U l        [        XS
SSS9U l
        g [        R                  " 5       U l        g )Nr   
upsample2d)       @r   znearest-exact)scale_factorr   r   r   rZ   
upsample3d)r   r   r   )r   r   r   downsample2d)r   r   r   r   )r   r   )rY   downsample3d)r   r   r   r\   )rY   rZ   )r   r   r/   r   rq   
Sequentialr   Conv2dresamplerV   	time_conv	ZeroPad2dIdentity)r"   r/   r   r   r#   s       r$   r   WanResample.__init__   s0   	 #"ax <MM/J		#A>DM \!MM/J		#A>DM -S'9iXDN^#MM",,|*DbiiPSZ[djFklDM^#MM",,|*DbiiPSZ[djFklDM,Sy\efDN KKMDMr&   r   c                 l   UR                  5       u  pEpgnU R                  S:X  Ga  UGb  US   n	X)   c  SX)'   US==   S-  ss'   GOUS S 2S S 2[        * S 2S S 2S S 24   R                  5       n
U
R                  S   S:  a^  X)   bY  X)   S:w  aQ  [
        R                  " X)   S S 2S S 2SS S 2S S 24   R                  S5      R                  U
R                  5      U
/SS9n
U
R                  S   S:  aQ  X)   bL  X)   S:X  aD  [
        R                  " [
        R                  " U
5      R                  U
R                  5      U
/SS9n
X)   S:X  a  U R                  U5      nOU R                  XU	   5      nXU	'   US==   S-  ss'   UR                  USXVXx5      n[
        R                  " US S 2SS S 2S S 2S S 2S S 24   US S 2SS S 2S S 2S S 2S S 24   4S5      nUR                  XEUS-  Xx5      nUR                  S   nUR                  SSSSS	5      R                  XF-  XWU5      nU R                  U5      nUR!                  XFUR                  S5      UR                  S5      UR                  S5      5      R                  SSSSS	5      nU R                  S
:X  a  Ub  US   n	X)   c!  UR                  5       X)'   US==   S-  ss'   U$ US S 2S S 2SS 2S S 2S S 24   R                  5       n
U R                  [
        R                  " X)   S S 2S S 2SS 2S S 2S S 24   U/S5      5      nXU	'   US==   S-  ss'   U$ )Nr   r   Repr   r   ry   r.   r   r,   r   )rQ   r   CACHE_Tcloner0   rD   rc   	unsqueezera   rb   
zeros_liker   reshapestackr4   r   r3   )r"   r'   
feat_cachefeat_idxbcthwidxrd   s              r$   r=   WanResample.forward	  s   aA99$%qk?*&+JOQK1$K1whiA 56<<>G}}Q'!+
0KPZP_chPh"'))'_Q2q!^<FFqILLW^^\^eflm# }}Q'!+
0KPZP_chPh"'))U-=-=g-F-I-I'..-Y[b,cij"k!%/ NN1- NN1o>&-sOQK1$K		!Qa3AQq!Q1a'7%8!Aq!Q1<L:M$NPQRA		!Aq4AGGAJIIaAq!$,,QUA!<MM!FF1AFF1Iqvvay9AA!Q1aP99&%qk?*&'ggiJOQK1$K 	  1bc1a0668Guyy*/!QQPQ/2RTU1VXY'Z[A&-sOQK1$Kr&   )r/   r   r   r   rf   )r@   rA   rB   rC   rg   rT   strr   r=   rF   rG   rH   s   @r$   r   r      sA    *C *s *c *T * *> %)A3 + +r&   r   c                   \   ^  \ rS rSrSr  SS\S\S\S\SS4
U 4S	 jjjrSS
/4S jr	Sr
U =r$ )WanResidualBlocki7  a)  
A custom residual block module.

Args:
    in_dim (int): Number of input channels.
    out_dim (int): Number of output channels.
    dropout (float, optional): Dropout rate for the dropout layer. Default is 0.0.
    non_linearity (str, optional): Type of non-linearity to use. Default is "silu".
in_dimout_dimdropoutnon_linearityr(   Nc                 n  > [         TU ]  5         Xl        X l        [	        U5      U l        [        USS9U l        [        XSSS9U l	        [        USS9U l
        [        R                  " U5      U l        [        X"SSS9U l        X:w  a  [        XS5      U l        g [        R                  " 5       U l        g )NFrl   r   r   r   )r   r   r   r   r   nonlinearityri   norm1rV   conv1norm2rq   Dropoutr   conv2r   conv_shortcut)r"   r   r   r   r   r#   s        r$   r   WanResidualBlock.__init__B  s     	*=9 !6
$VaC
 7
zz'*$Wq!D
DJDU_Va@[][f[f[hr&   r   c           	         U R                  U5      nU R                  U5      nU R                  U5      nUb  US   nUS S 2S S 2[        * S 2S S 2S S 24   R	                  5       nUR
                  S   S:  aV  X%   bQ  [        R                  " X%   S S 2S S 2SS S 2S S 24   R                  S5      R                  UR                  5      U/SS9nU R                  XU   5      nXbU'   US==   S-  ss'   OU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nUb  US   nUS S 2S S 2[        * S 2S S 2S S 24   R	                  5       nUR
                  S   S:  aV  X%   bQ  [        R                  " X%   S S 2S S 2SS S 2S S 24   R                  S5      R                  UR                  5      U/SS9nU R                  XU   5      nXbU'   US==   S-  ss'   X-   $ U R                  U5      nX-   $ Nr   r   ry   r.   r   )r   r   r   r   r   r0   rD   rc   r   ra   rb   r   r   r   r   )r"   r'   r   r   r   r   rd   s          r$   r=   WanResidualBlock.forwardV  s   q! JJqMa !1+C1whiA-.446G}}Q!#
(C))Z_Q2q!^%D%N%Nq%Q%T%TU\UcUc%dfm$ntuv

1o.A%sOQK1K

1A JJqMa  LLO!1+C1whiA-.446G}}Q!#
(C))Z_Q2q!^%D%N%Nq%Q%T%TU\UcUc%dfm$ntuv

1o.A%sOQK1K
 u 

1A ur&   )	r   r   r   r   r   r   r   r   r   )ro   silur@   rA   rB   rC   rg   rT   r   r   r   r=   rF   rG   rH   s   @r$   r   r   7  sd     #ii i 	i
 i 
i i( %)A3 ( (r&   r   c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )WanAttentionBlocki  zm
Causal self-attention with a single head.

Args:
    dim (int): The number of channels in the input tensor.
c                    > [         TU ]  5         Xl        [        U5      U l        [
        R                  " XS-  S5      U l        [
        R                  " XS5      U l        g )Nr   r   )	r   r   r/   ri   normrq   r   to_qkvproj)r"   r/   r#   s     r$   r   WanAttentionBlock.__init__  sI      $	ii1Wa0IIc*	r&   c                 p   UnUR                  5       u  p4pVnUR                  SSSSS5      R                  X5-  XFU5      nU R                  U5      nU R	                  U5      nUR                  X5-  SUS-  S5      nUR                  SSSS5      R                  5       nUR                  SSS9u  pn[        R                  " XU5      nUR                  S5      R                  SSS5      R                  X5-  XFU5      nU R                  U5      nUR                  X5XFU5      nUR                  SSSSS5      nX-   $ )Nr   r   r   r   r,   ry   r.   )rQ   r4   r   r   r   r5   chunkr1   scaled_dot_product_attentionsqueezer   r3   )r"   r'   identity
batch_sizechannelstimeheightwidthqkvqkvs               r$   r=   WanAttentionBlock.forward  s0   45FFH1
dEIIaAq!$,,Z->RWXIIaL kk!nkk*+Q1bAkk!Q1%002))A2)&a **13IIaL  Aq)11*2CXW\] IIaL FF:Xu=IIaAq!$|r&   )r/   r   r   r   )	r@   rA   rB   rC   rg   r   r=   rF   rG   rH   s   @r$   r   r     s    + r&   r   c            	       T   ^  \ rS rSrSrSS\S\S\S\4U 4S jjjrSS	/4S
 jr	Sr
U =r$ )WanMidBlocki  z
Middle block for WanVAE encoder and decoder.

Args:
    dim (int): Number of input/output channels.
    dropout (float): Dropout rate.
    non_linearity (str): Type of non-linearity to use.
r/   r   r   
num_layersc           	      T  > [         TU ]  5         Xl        [        XX#5      /n/ n[	        U5       H8  nUR                  [        U5      5        UR                  [        XX#5      5        M:     [        R                  " U5      U l	        [        R                  " U5      U l
        SU l        g )NF)r   r   r/   r   rangeappendr   rq   
ModuleList
attentionsresnetsgradient_checkpointing)	r"   r/   r   r   r   r   r   _r#   s	           r$   r   WanMidBlock.__init__  s     $CgEF
z"A/45NN+CgMN # --
3}}W-&+#r&   Nr   c                     U R                   S   " XU5      n[        U R                  U R                   SS  5       H  u  pEUb  U" U5      nU" XU5      nM     U$ )Nr   r   )r   zipr   )r"   r'   r   r   attnresnets         r$   r=   WanMidBlock.forward  s]    LLOA84  ab1ABLDGqh/A	 C r&   )r   r/   r   r   )ro   r   r   r   rH   s   @r$   r   r     sB    ,C ,% ,c ,`c , , %)A3  r&   r   c                   <   ^  \ rS rSrSU 4S jjrSS/4S jrSrU =r$ )WanResidualDownBlocki  c                 P  > [         T
U ]  5         [        UUU(       a  SOSU(       a  SOSS9U l        / n[	        U5       H   nUR                  [        XU5      5        UnM"     [        R                  " U5      U l	        U(       a  U(       a  SOSn	[        X)S9U l        g S U l        g )Nr   r   r   r   r   r   r   )r   r   r   avg_shortcutr   r   r   rq   r   r   r   downsampler)r"   r   r   r   num_res_blockstemperal_downsample	down_flagr   r   r   r#   s             r$   r   WanResidualDownBlock.__init__  s     &-Q1#Q	
 ~&ANN+FWEFF ' }}W- %8>nD*7>D#Dr&   Nr   c                     UR                  5       nU R                   H  nU" XU5      nM     U R                  b  U R                  XU5      nXR                  U5      -   $ rf   )r   r   r   r   )r"   r'   r   r   x_copyr   s         r$   r=   WanResidualDownBlock.forward  sY    llFqh/A #'  9A$$V,,,r&   )r   r   r   )FF)r@   rA   rB   rC   r   r=   rF   rG   rH   s   @r$   r   r     s    $2 %)A3 - -r&   r   c            
       l   ^  \ rS rSrSrSSS/ SQS/ / SQS	S
S4
S\S\S\4U 4S jjjrSS/4S jr	Sr
U =r$ )WanEncoder3di  aC  
A 3D encoder module.

Args:
    dim (int): The base number of channels in the first layer.
    z_dim (int): The dimensionality of the latent space.
    dim_mult (list of int): Multipliers for the number of channels in each block.
    num_res_blocks (int): Number of residual blocks in each block.
    attn_scales (list of float): Scales at which to apply attention mechanisms.
    temperal_downsample (list of bool): Whether to downsample temporally in each block.
    dropout (float): Dropout rate for the dropout layers.
    non_linearity (str): Type of non-linearity to use.
r      r,   r   r   r,   r,   r   r|   ro   r   Fr   r   is_residualc                   > [         TU ]  5         X l        X0l        X@l        XPl        X`l        Xpl        [        U	5      U l	        S/U-    Vs/ s H  oU-  PM	     nnSn[        XS   SSS9U l        [        R                  " / 5      U l        [        [!        US S USS  5      5       GH  u  nu  nnU
(       aN  U R                  R#                  [%        UUUUU['        U5      S-
  :w  a  X~   OSU['        U5      S-
  :g  S95        M_  [)        U5       HT  nU R                  R#                  [+        UUU5      5        X;   a$  U R                  R#                  [-        U5      5        UnMV     U['        U5      S-
  :w  d  M  X~   (       a  S	OS
nU R                  R#                  [/        UUS95        US-  nGM     [1        WXSS9U l        [5        USS9U l        [        UUSSS9U l        SU l        g s  snf )Nr         ?r   r   r   ry   F)r   r   r   r   r   r   r   r   )r   r   r/   z_dimdim_multr   attn_scalesr   r   r   rV   conv_inrq   r   down_blocks	enumerater   r   r   lenr   r   r   r   r   	mid_blockri   norm_outconv_outr   )r"   r   r/   r   r   r   r   r   r   r   r   udimsrp   ir   r   r   r   r#   s                      r$   r   WanEncoder3d.__init__  s    	
 ,&#6 *=9 #$x0Aa0 '{GQJ ==,$-c$s)T!"X.F$G A   ''(&FG3x=[\K\F\,?,Bbg"#s8}q'8"8	 ~.A$$++,<VWg,VW+((//0A'0JK$F	 / H))-@-C>D$$++Kd,KLSLE1 %H6 %WgQRS $GE:'1E&+#S 1s   G/Nr   c           	         Ub  US   nUS S 2S S 2[         * S 2S S 2S S 24   R                  5       nUR                  S   S:  aV  X$   bQ  [        R                  " X$   S S 2S S 2SS S 2S S 24   R                  S5      R                  UR                  5      U/SS9nU R                  XU   5      nXRU'   US==   S-  ss'   OU R                  U5      nU R                   H  nUb  U" XU5      nM  U" U5      nM     U R                  XU5      nU R                  U5      nU R                  U5      nUb  US   nUS S 2S S 2[         * S 2S S 2S S 24   R                  5       nUR                  S   S:  aV  X$   bQ  [        R                  " X$   S S 2S S 2SS S 2S S 24   R                  S5      R                  UR                  5      U/SS9nU R                  XU   5      nXRU'   US==   S-  ss'   U$ U R                  U5      nU$ r   )r   r   r0   rD   rc   r   ra   rb   r  r  r  r  r   r  )r"   r'   r   r   r   rd   layers          r$   r=   WanEncoder3d.forwardJ  s   !1+C1whiA-.446G}}Q!#
(C))Z_Q2q!^%D%N%Nq%Q%T%TU\UcUc%dfm$ntuvQ30A%sOQK1KQA %%E%!2!H	 & NN1(3 MM!a !1+C1whiA-.446G}}Q!#
(C))Z_Q2q!^%D%N%Nq%Q%T%TU\UcUc%dfm$ntuvaC1A%sOQK1K  a Ar&   )r   r  r  r/   r   r  r   r  r   r  r   r   r   )r@   rA   rB   rC   rg   rT   r   r}   r   r=   rF   rG   rH   s   @r$   r   r     sg      /#!@,@, @, @, @,D %)A3 % %r&   r   c                   j   ^  \ rS rSrSr    SS\S\S\S\S\S	\S
\4U 4S jjjr	SS/S4S jr
SrU =r$ )WanResidualUpBlockir  a  
A block that handles upsampling for the WanVAE decoder.

Args:
    in_dim (int): Input dimension
    out_dim (int): Output dimension
    num_res_blocks (int): Number of residual blocks
    dropout (float): Dropout rate
    temperal_upsample (bool): Whether to upsample on temporal dimension
    up_flag (bool): Whether to upsample or not
    non_linearity (str): Type of non-linearity to use
Fr   r   r   r   temperal_upsampleup_flagr   c           	        > [         TU ]  5         Xl        X l        U(       a  [	        UUU(       a  SOSSS9U l        OS U l        / nUn	[        US-   5       H   n
UR                  [        XXG5      5        Un	M"     [        R                  " U5      U l        U(       a  U(       a  SOSn[        X+US9U l        OS U l        SU l        g )Nr   r   r   r   r   )r   r   F)r   r   r   r   rJ   r   r   r   r   rq   r   r   r   	upsamplerr   )r"   r   r   r   r   r  r  r   r   current_dimr   upsample_moder#   s               r$   r   WanResidualUpBlock.__init__  s     	 '/Q	!D !%D ~)*ANN+K'YZ!K + }}W- ,=L<M(W^_DN!DN&+#r&   Nr   c                    UR                  5       nU R                   H  nUb  U" XU5      nM  U" U5      nM     U R                  b'  Ub  U R                  XU5      nOU R                  U5      nU R                  b  XR                  XTS9-   nU$ )	  
Forward pass through the upsampling block.

Args:
    x (torch.Tensor): Input tensor
    feat_cache (list, optional): Feature cache for causal convolutions
    feat_idx (list, optional): Feature index for cache management

Returns:
    torch.Tensor: Output tensor
rR   )r   r   r  r   )r"   r'   r   r   rR   r   r   s          r$   r=   WanResidualUpBlock.forward  s     llF%1(31I	 # >>%%NN1(;NN1%(%%f%FFAr&   )r   r   r   r   r   r  )ro   FFr   )r@   rA   rB   rC   rg   rT   r   r}   r   r   r=   rF   rG   rH   s   @r$   r  r  r  sz    $ "'#(,(, (, 	(,
 (,  (, (, (, (,T %)A3E  r&   r  c                   j   ^  \ rS rSrSr   SS\S\S\S\S\\   S	\4U 4S
 jjjr	SS/S4S jr
SrU =r$ )
WanUpBlocki  ah  
A block that handles upsampling for the WanVAE decoder.

Args:
    in_dim (int): Input dimension
    out_dim (int): Output dimension
    num_res_blocks (int): Number of residual blocks
    dropout (float): Dropout rate
    upsample_mode (str, optional): Mode for upsampling ('upsample2d' or 'upsample3d')
    non_linearity (str): Type of non-linearity to use
Nr   r   r   r   r  r   c           	      D  > [         T
U ]  5         Xl        X l        / nUn[	        US-   5       H   n	UR                  [        XXF5      5        UnM"     [        R                  " U5      U l	        S U l
        Ub#  [        R                  " [        X%S9/5      U l
        SU l        g )Nr   r   F)r   r   r   r   r   r   r   rq   r   r   
upsamplersr   r   )r"   r   r   r   r   r  r   r   r  r   r#   s             r$   r   WanUpBlock.__init__  s     	 ~)*ANN+K'YZ!K + }}W- $ mm[-U,VWDO&+#r&   r   c                     U R                    H  nUb  U" XU5      nM  U" U5      nM     U R                  b0  Ub  U R                  S   " XU5      nU$ U R                  S   " U5      nU$ )r  r   )r   r  )r"   r'   r   r   rR   r   s         r$   r=   WanUpBlock.forward  su     llF%1(31I	 # ??&%OOA&qh?  OOA&q)r&   )r   r   r   r   r  )ro   Nr   )r@   rA   rB   rC   rg   rT   r   r   r   r   r=   rF   rG   rH   s   @r$   r  r    sp    
" '+#,, , 	,
 ,  }, , ,< %)A3D  r&   r  c            
       n   ^  \ rS rSrSrSS/ SQS/ / SQSS	S
S4
S\S\S\4U 4S jjjrSS/S4S jr	Sr
U =r$ )WanDecoder3di  a?  
A 3D decoder module.

Args:
    dim (int): The base number of channels in the first layer.
    z_dim (int): The dimensionality of the latent space.
    dim_mult (list of int): Multipliers for the number of channels in each block.
    num_res_blocks (int): Number of residual blocks in each block.
    attn_scales (list of float): Scales at which to apply attention mechanisms.
    temperal_upsample (list of bool): Whether to upsample temporally in each block.
    dropout (float): Dropout rate for the dropout layers.
    non_linearity (str): Type of non-linearity to use.
r   r,   r   r   FTTro   r   r   Fr   r   r   c                 
  > [         TU ]  5         Xl        X l        X0l        X@l        XPl        X`l        [        U5      U l	        US   /US S S2   -    Vs/ s H  oU-  PM	     nn[        X,S   SSS9U l        [        US   XxSS9U l        [        R                  " / 5      U l        [#        [%        US S USS  5      5       H  u  nu  pUS:  a  U
(       d  US-  nU['        U5      S-
  :g  nS nU(       a  Xm   (       a  SnO	U(       a  S	nU
(       a  [)        UUUUU(       a  Xm   OS
UUS9nO[+        UUUUUUS9nU R                   R-                  U5        M     [/        WS
S9U l        [        XSSS9U l        S
U l        g s  snf )Nry   r   r   r   r   r   r   r   r   F)r   r   r   r   r  r  r   )r   r   r   r   r  r   r   )r   r   r/   r   r   r   r   r  r   r   rV   r  r   r  rq   r   	up_blocksr  r   r  r  r  r   ri   r  r  r   )r"   r/   r   r   r   r   r  r   r   r   r   r  r	  r
  r   r   r  r  up_blockr#   s                      r$   r   WanDecoder3d.__init__  s    	
 ,&!2*=9 #+2,(4R4.!@A!@Aa!@A 'u1gq!D %T!WgQRS r*$-c$s)T!"X.F$G A 1u[1 3x=1,,G M,/ , ,-!##1#>E&7&:5#"/ &!##1#"/"/ NN!!(+E %HJ $GE:'q!L&+#e Bs   F Nr   c           	         Ub  US   nUS S 2S S 2[         * S 2S S 2S S 24   R                  5       nUR                  S   S:  aV  X%   bQ  [        R                  " X%   S S 2S S 2SS S 2S S 24   R                  S5      R                  UR                  5      U/SS9nU R                  XU   5      nXbU'   US==   S-  ss'   OU R                  U5      nU R                  XU5      nU R                   H
  nU" XX4S9nM     U R                  U5      nU R                  U5      nUb  US   nUS S 2S S 2[         * S 2S S 2S S 24   R                  5       nUR                  S   S:  aV  X%   bQ  [        R                  " X%   S S 2S S 2SS S 2S S 24   R                  S5      R                  UR                  5      U/SS9nU R                  XU   5      nXbU'   US==   S-  ss'   U$ U R                  U5      nU$ )Nr   r   ry   r.   r   r  )r   r   r0   rD   rc   r   ra   rb   r  r  r'  r  r   r  )r"   r'   r   r   rR   r   rd   r(  s           r$   r=   WanDecoder3d.forwardj  s   !1+C1whiA-.446G}}Q!#
(C))Z_Q2q!^%D%N%Nq%Q%T%TU\UcUc%dfm$ntuvQ30A%sOQK1KQA NN1(3 HJA ' MM!a !1+C1whiA-.446G}}Q!#
(C))Z_Q2q!^%D%N%Nq%Q%T%TU\UcUc%dfm$ntuvaC1A%sOQK1K  a Ar&   )r   r  r  r/   r   r   r  r   r  r   r  r'  r   )r@   rA   rB   rC   rg   r   rT   r}   r   r=   rF   rG   rH   s   @r$   r$  r$    sk      -#!J, J, J, J, J,X %)A3E # #r&   r$  c           	         US:X  a  U $ U R                  5       S:w  a  [        SU R                   35      eU R                  u  p#pEnXQ-  S:w  d  Xa-  S:w  a  [        SU SU SU S35      eU R                  X#XEU-  XU-  U5      n U R	                  SSS	S
SSS5      R                  5       n U R                  X#U-  U-  XEU-  Xa-  5      n U $ )Nr   r*   Invalid input shape: r   zHeight (z) and width (z#) must be divisible by patch_size ()r-   r,   r   r   r/   
ValueErrorr0   r3   r4   r5   )r'   
patch_sizer   r   framesr   r   s          r$   patchifyr3    s    Quuw!|0	:;;23''/J&% a5#5#:8F8=?bcmbnnopqq 	
zVz-A:XbObdnoA 	
		!Q1aA&113A	zj0:=vQ[G[]b]pqAHr&   c           	      >   US:X  a  U $ U R                  5       S:w  a  [        SU R                   35      eU R                  u  p#pEnX1U-  -  nU R                  X'XXEU5      n U R	                  SSSSSSS5      R                  5       n U R                  X'XEU-  Xa-  5      n U $ )	Nr   r*   r-  r   r,   r   r-   r   r/  )r'   r1  r   	c_patchesr2  r   r   r   s           r$   
unpatchifyr6    s    Quuw!|0	:;;34770J65*45H 	
zZVUSA 	
		!Q1aA&113A	zVj-@%BTUAHr&   c            $         ^  \ rS rSrSrSr\SSS/ SQS/ / S	QS
/ SQ/ SQSSSSSS4S\S\\   S\S\	\   S\S\
\   S\
\   S\S\
\   S\
\   S\S\S\S\\   S\\   S\\   S S4"U 4S! jjj5       r    S>S"\\   S#\\   S$\\   S%\\   S S4
S& jjrS?S' jrS?S( jrS?S) jrS* rS+\R(                  4S, jr\ S@S+\R(                  S-\S \\\	\   4   4S. jj5       rS@S/\R(                  S-\4S0 jjr\S@S/\R(                  S-\S \\\R(                  4   4S1 jj5       rS2\R(                  S3\R(                  S4\S \R(                  4S5 jrS2\R(                  S3\R(                  S4\S \R(                  4S6 jrS+\R(                  S \4S7 jr S@S/\R(                  S-\S \\\R(                  4   4S8 jjr!   SAS9\R(                  S:\S-\S;\\RD                     S \\\R(                  4   4
S< jjr#S=r$U =r%$ )BAutoencoderKLWani  a'  
A VAE model with KL loss for encoding videos into latents and decoding latent representations into videos.
Introduced in [Wan 2.1].

This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented
for all models (such as downloading or saving).
F`   N   r   r   r%  ro   )gy):gMOg^)gQ?gtVƿgZӼ?gBfjÿgU0*?gL
F%u?gMg&?gz6>׿gF%uȿg[ AcgMJ?gW2ıҿ)g_L@gNё\C?gQ@g?@g9#J{?g|a2U?gHPs@g0* @gJ{/L&
@gJY8@g]C@g(?gK46?gS:?go_Ι@g-?r   r,      base_dimdecoder_base_dimr   r   r   r   r   r   latents_meanlatents_stdr   r   r   r1  scale_factor_temporalscale_factor_spatialr(   c                   > [         TU ]  5         X0l        Xpl        US S S2   U l        Uc  Un[        UUUS-  UUUUUUS9	U l        [        US-  US-  S5      U l        [        X3S5      U l	        [        UUUUUU R                  UUUS9	U l        S[        U R                  5      -  U l        SU l        SU l        SU l        SU l        SU l        SU l        U R                  b*  [)        S	 U R                  R+                  5        5       5      OS
U R                  b*  [)        S U R                  R+                  5        5       5      OS
S.U l        g )Nry   r   )	r   r/   r   r   r   r   r   r   r   r   )	r/   r   r   r   r   r  r   r   r   F      c              3   B   #    U  H  n[        U[        5      v   M     g 7frf   
isinstancerV   .0ms     r$   	<genexpr>,AutoencoderKLWan.__init__.<locals>.<genexpr>4       ZCYa:a99CY   r   c              3   B   #    U  H  n[        U[        5      v   M     g 7frf   rF  rH  s     r$   rK  rL  7  rM  rN  )decoderencoder)r   r   r   r   r  r   rQ  rV   
quant_convpost_quant_convr$  rP  r  spatial_compression_ratiouse_slicing
use_tilingtile_sample_min_heighttile_sample_min_widthtile_sample_stride_heighttile_sample_stride_widthsummodules_cached_conv_counts)r"   r<  r=  r   r   r   r   r   r   r>  r?  r   r   r   r1  r@  rA  r#   s                    r$   r   AutoencoderKLWan.__init__  sk   l 	
#6 !4TrT!:#'##!))# 3#

 *%!)UQYB.uQ?# )#"44%#

 *+c$2J2J.K)K& !
   '*#%(" *-&(+%
 ||' Z4<<CWCWCYZZ||' Z4<<CWCWCYZZ$
 r&   rW  rX  rY  rZ  c                     SU l         U=(       d    U R                  U l        U=(       d    U R                  U l        U=(       d    U R                  U l        U=(       d    U R                  U l        g)a  
Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
processing larger images.

Args:
    tile_sample_min_height (`int`, *optional*):
        The minimum height required for a sample to be separated into tiles across the height dimension.
    tile_sample_min_width (`int`, *optional*):
        The minimum width required for a sample to be separated into tiles across the width dimension.
    tile_sample_stride_height (`int`, *optional*):
        The minimum amount of overlap between two consecutive vertical tiles. This is to ensure that there are
        no tiling artifacts produced across the height dimension.
    tile_sample_stride_width (`int`, *optional*):
        The stride between two consecutive horizontal tiles. This is to ensure that there are no tiling
        artifacts produced across the width dimension.
TN)rV  rW  rX  rY  rZ  )r"   rW  rX  rY  rZ  s        r$   enable_tilingAutoencoderKLWan.enable_tiling<  sW    0 &<&[@[@[#%:%Xd>X>X")B)ddFdFd&(@(aDDaDa%r&   c                     SU l         g)z
Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing
decoding in one step.
FN)rV  r"   s    r$   disable_tilingAutoencoderKLWan.disable_tilingZ  s    
  r&   c                     SU l         g)z
Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.
TNrU  rc  s    r$   enable_slicingAutoencoderKLWan.enable_slicinga  s    
  r&   c                     SU l         g)z
Disable sliced VAE decoding. If `enable_slicing` was previously enabled, this method will go back to computing
decoding in one step.
FNrg  rc  s    r$   disable_slicing AutoencoderKLWan.disable_slicingh  s    
 !r&   c                     U R                   S   U l        S/U l        S /U R                  -  U l        U R                   S   U l        S/U l        S /U R                  -  U l        g )NrP  r   rQ  )r]  	_conv_num	_conv_idx	_feat_map_enc_conv_num_enc_conv_idx_enc_feat_maprc  s    r$   clear_cacheAutoencoderKLWan.clear_cacheo  sa    11)<$..0!55i@S"Vd&8&88r&   r'   c           
         UR                   u    p#pEU R                  (       a/  XPR                  :  d  X@R                  :  a  U R	                  U5      $ U R                  5         U R                  R                  b  [        XR                  R                  S9nSUS-
  S-  -   n[        U5       H  nS/U l
        US:X  a9  U R                  US S 2S S 2S S2S S 2S S 24   U R                  U R                  S9nMJ  U R                  US S 2S S 2SSUS-
  -  -   SSU-  -   2S S 2S S 24   U R                  U R                  S9n	[        R                  " WU	/S5      nM     U R                  W5      n
U R                  5         U
$ )Nr1  r   r,   r   r   r   r   )r0   rV  rX  rW  tiled_encodert  configr1  r3  r   rr  rQ  rs  rD   rc   rR  )r"   r'   r   	num_framer   r   iter_r
  outout_encs              r$   _encodeAutoencoderKLWan._encodey  s^   )*&1??(B(B BfOjOjFj$$Q'';;!!-{{'='=>AY]q((uA"#DAvll1Q2A2q!^#4ASAS^b^p^plq||aAQUOa!a%i7A=>#11!// $ 
 iidQ/  ooc"
r&   return_dictc                 >   U R                   (       aY  UR                  S   S:  aF  UR                  S5       Vs/ s H  o0R                  U5      PM     nn[        R
                  " U5      nOU R                  U5      n[        U5      nU(       d  U4$ [        US9$ s  snf )a  
Encode a batch of images into latents.

Args:
    x (`torch.Tensor`): Input batch of images.
    return_dict (`bool`, *optional*, defaults to `True`):
        Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple.

Returns:
        The latent representations of the encoded videos. If `return_dict` is True, a
        [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned.
r   r   )latent_dist)rU  r0   splitr  rD   rc   r   r   )r"   r'   r  x_sliceencoded_slicesr   	posteriors          r$   encodeAutoencoderKLWan.encode  s      
QCD771:N:ll73:NN		.)AQA03	<"y99 Os   Bzc           
      T   UR                   u    p4pVU R                  U R                  -  nU R                  U R                  -  nU R                  (       a  Xh:  d  XW:  a  U R                  XS9$ U R                  5         U R                  U5      n	[        U5       H  n
S/U l	        U
S:X  a<  U R                  U	S S 2S S 2XS-   2S S 2S S 24   U R                  U R                  SS9nMM  U R                  U	S S 2S S 2XS-   2S S 2S S 24   U R                  U R                  S9n[        R                  " WU/S5      nM     U R                  R                  b  [!        WU R                  R                  S9n[        R"                  " WS	S
S9nU R                  5         U(       d  U4$ [%        US9$ )Nr  r   r   T)r   r   rR   rx  r   rw  g      r   )minmaxsample)r0   rW  rT  rX  rV  tiled_decodert  rS  r   ro  rP  rp  rD   rc   rz  r1  r6  clampr   )r"   r  r  r   r{  r   r   tile_latent_min_heighttile_latent_min_widthr'   r
  r}  r~  s                r$   _decodeAutoencoderKLWan._decode  s   )*&1!%!<!<@^@^!^ $ : :d>\>\ \?? =A`$$Q$@@  #y!ASDNAvllaAAIq!+,RVR`R`nr #  ||AaAAIq!&;$<bfbpbp|qiidQ/ " ;;!!-ST[[-C-CDCkk#4S16MC((r&   c                 P   U R                   (       ac  UR                  S   S:  aP  UR                  S5       Vs/ s H  o0R                  U5      R                  PM     nn[
        R                  " U5      nOU R                  U5      R                  nU(       d  U4$ [        US9$ s  snf )a  
Decode a batch of images.

Args:
    z (`torch.Tensor`): Input batch of latent vectors.
    return_dict (`bool`, *optional*, defaults to `True`):
        Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple.

Returns:
    [`~models.vae.DecoderOutput`] or `tuple`:
        If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is
        returned.
r   r   r  )rU  r0   r  r  r  rD   rc   r   )r"   r  r  z_slicedecoded_slicesdecodeds         r$   decodeAutoencoderKLWan.decode  s     
QJK''RS*U*wll73::*NUii/Gll1o,,G:G,, Vs   $B#ar   blend_extentc           	         [        UR                  S   UR                  S   U5      n[        U5       HI  nUS S 2S S 2S S 2U* U-   S S 24   SXC-  -
  -  US S 2S S 2S S 2US S 24   XC-  -  -   US S 2S S 2S S 2US S 24'   MK     U$ )Nr   r  r0   r   )r"   r  r   r  ys        r$   blend_vAutoencoderKLWan.blend_v  s    1772;\B|$A Aq<-!*;Q!>?1qGWCWX[\]^`acdfgij]j[k \  AaAq!m % r&   c                    [        UR                  S   UR                  S   U5      n[        U5       HI  nUS S 2S S 2S S 2S S 2U* U-   4   SXC-  -
  -  US S 2S S 2S S 2S S 2U4   XC-  -  -   US S 2S S 2S S 2S S 2U4'   MK     U$ )Nry   r   r  )r"   r  r   r  r'   s        r$   blend_hAutoencoderKLWan.blend_h  s    1772;\B|$A Aq!l]Q->!>?1qGWCWX[\]^`acdfgij]j[k \  AaAq!m % r&   c                    UR                   u    p#pEX@R                  -  nXPR                  -  nU R                  U R                  -  nU R                  U R                  -  n	U R                  U R                  -  n
U R
                  U R                  -  nX-
  nX-
  n/ n[        SX@R                  5       GHH  n/ n[        SXPR
                  5       GH  nU R                  5         / nSUS-
  S-  -   n[        U5       H  nS/U l        US:X  a.  USS2SS2SS2XU R                  -   2UUU R                  -   24   nO<USS2SS2SSUS-
  -  -   SSU-  -   2XU R                  -   2UUU R                  -   24   nU R                  UU R                  U R                  S9nU R                  U5      nUR                  U5        M     UR                  [        R                  " USS95        GM     UR                  U5        GMK     U R                  5         / n[        U5       H  u  nn/ n[        U5       Hi  u  nnUS:  a  U R!                  XS-
     U   UU5      nUS:  a  U R#                  UUS-
     UU5      nUR                  USS2SS2SS2SU
2SU24   5        Mk     UR                  [        R                  " USS95        M     [        R                  " US	S9SS2SS2SS2SU2SU24   nU$ )
zEncode a batch of images using a tiled encoder.

Args:
    x (`torch.Tensor`): Input batch of videos.

Returns:
    `torch.Tensor`:
        The latent representation of the encoded videos.
r   r   r,   Nrx  r   r.   ry   r   )r0   rT  rW  rX  rY  rZ  r   rt  rr  rQ  rs  rR  r   rD   rc   r  r  r  )r"   r'   r   
num_framesr   r   latent_heightlatent_widthr  r  tile_latent_stride_heighttile_latent_stride_widthblend_heightblend_widthrowsr
  rowjr   frame_ranger   tileresult_rows
result_rowr  s                            r$   ry  AutoencoderKLWan.tiled_encode  s    +,'''1&"@"@@ > >>!%!<!<@^@^!^ $ : :d>\>\ \$($B$BdFdFd$d!#'#@#@DDbDb#b -I+F q&"@"@AAC1e%B%BC  ":>a"77{+A*+D&Av Arr143N3N/N+NPQTUX\XrXrTrPr!rs QUOa!a%i7D$?$? ??D$>$> >>	@   <<9K9KVZVhVh<iD??40DKK% , 

599Tq12' D( KK- B. 	oFAsJ$S>4 q5<<UAlKDq5<<AE
D+FD!!$q!Q0J1J0JLeMeLe'e"fg * uyy<= & ii+Aq!^m^]l],RS
r&   c                    UR                   u    p4pVXPR                  -  nX`R                  -  nU R                  U R                  -  n	U R                  U R                  -  n
U R                  U R                  -  nU R
                  U R                  -  nU R                  U R                  -
  nU R                  U R
                  -
  n/ n[        SX[5       H  n/ n[        SXl5       H  nU R                  5         / n[        U5       Ho  nS/U l        USS2SS2UUS-   2UUU	-   2UUU
-   24   nU R                  U5      nU R                  UU R                  U R                  S9nUR                  U5        Mq     UR                  [        R                  " USS95        M     UR                  U5        M     U R                  5         / n[        U5       H  u  nn/ n[        U5       H~  u  nnUS:  a  U R!                  UUS-
     U   UU5      nUS:  a  U R#                  UUS-
     UU5      nUR                  USS2SS2SS2SU R                  2SU R
                  24   5        M     UR                  [        R                  " USS95        M     [        R                  " USS9SS2SS2SS2SU2SU24   nU(       d  U4$ [%        US	9$ )
a  
Decode a batch of images using a tiled decoder.

Args:
    z (`torch.Tensor`): Input batch of latent vectors.
    return_dict (`bool`, *optional*, defaults to `True`):
        Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple.

Returns:
    [`~models.vae.DecoderOutput`] or `tuple`:
        If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is
        returned.
r   Nr   rx  r   r.   ry   r   r  )r0   rT  rW  rX  rY  rZ  r   rt  ro  rS  rP  rp  r   rD   rc   r  r  r  r   )r"   r  r  r   r  r   r   sample_heightsample_widthr  r  r  r  r  r  r  r
  r  r  r   r   r  r  r  r  decs                             r$   r  AutoencoderKLWan.tiled_decode8  s    +,'''1&!?!??===!%!<!<@^@^!^ $ : :d>\>\ \$($B$BdFdFd$d!#'#@#@DDbDb#b 22T5S5SS0043P3PP q&<AC1e>  "z*A&'SDNQ1q1u9a!6L2L.LaRSVkRkNkklD//5D"ll4DNNUYUcUcldGKK( + 

599Tq12 ? KK = 	oFAsJ$S>4 q5<<QUAlKDq5<<AE
D+FD!!$q!Q0P$2P2P0PRqTXTqTqRq'q"rs * uyy<= & ii+Aq!^m^]l],RS6MC((r&   r  sample_posterior	generatorc                     UnU R                  U5      R                  nU(       a  UR                  US9nOUR                  5       nU R	                  XsS9nU$ )z
Args:
    sample (`torch.Tensor`): Input sample.
    return_dict (`bool`, *optional*, defaults to `True`):
        Whether or not to return a [`DecoderOutput`] instead of a plain tuple.
)r  r  )r  r  r  r   r  )	r"   r  r  r  r  r'   r  r  r  s	            r$   r=   AutoencoderKLWan.forwardw  sS     KKN..	  9 5A Akk!k5
r&   )r]  ro  rn  rr  rq  rs  rp  rP  rQ  rS  rR  rT  r   r  rW  rX  rY  rZ  rU  rV  r   )NNNN)r(   N)T)FTN)&r@   rA   rB   rC   rg    _supports_gradient_checkpointingr	   rT   r   r   r   r   r}   r   r`  rd  rh  rk  rt  rD   rE   r  r   r   r   r   r  r  r   r  r  r  ry  r  	Generatorr=   rF   rG   rH   s   @r$   r8  r8    se    (-$ *.+#%*=%
$$
$ "$(/0./gs
s
 #3-s
 	s

 *s
 s
 %[s
 "$Zs
 s
 5ks
8 %[9s
\ ]s
^ _s
` as
b SMcs
d  (}es
f 'smgs
h 
is
 s
n 15/35948b (b  (}b $,E?	b
 #+5/b 
b<  !9 4 37::,0:	"E*F$GG	H: :4) )D )> - -4 -5X]XdXdIdCe - -0 %,, c ell  %,, c ell @ell @/B @D=)ell =) =)}^c^j^jOjIk =)D "' /3  	
 EOO, 
}ell*	+ r&   r8  )6typingr   r   r   r   rD   torch.nnrq   torch.nn.functional
functionalr1   torch.utils.checkpointconfiguration_utilsr   r	   loadersr
   utilsr   utils.accelerate_utilsr   activationsr   modeling_outputsr   modeling_utilsr   vaer   r   
get_loggerr@   loggerr   Moduler   rJ   Conv3drV   ri   Upsampler   r   r   r   r   r   r   r  r  r$  r3  r6  r8  r   r&   r$   <module>r     sb   0 /      B -  8 ( 2 ' < 
		H	%
2		 2j)bii )X*"bii *"Zk")) k45"++ 5X")) XvGryy GT*		 *Z$")) $N!-299 !-Hv299 vrU UpB BJ~299 ~B.(Pz;0F Pr&   