
    +hhP                     <   S SK Jr  S SKJrJrJr  S SKrS SKrS SKJr  SSK	J
r
Jr  SSKJrJr  SS	KJrJrJr  SS
KJrJr  SSKJr  \R0                  " \5      r\ " S S\5      5       r " S S\R8                  5      r " S S\\
5      r " S S\R8                  5      r " S S\R8                  5      r  " S S\R8                  5      r! " S S\R8                  5      r" " S S\R8                  5      r# " S S\R8                  5      r$ " S S\R8                  5      r%g)     )	dataclass)DictTupleUnionN)nn   )ConfigMixinregister_to_config)
BaseOutputlogging   )	AttentionAttentionProcessorAttnProcessor)TimestepEmbedding	Timesteps)
ModelMixinc                   8    \ rS rSr% Sr\R                  \S'   Srg)Kandinsky3UNetOutput    Nsample )	__name__
__module____qualname____firstlineno__r   torchTensor__annotations____static_attributes__r       `/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/models/unets/unet_kandinsky3.pyr   r       s    FELLr!   r   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )Kandinsky3EncoderProj%   c                    > [         TU ]  5         [        R                  " XSS9U l        [        R
                  " U5      U l        g )NF)bias)super__init__r   Linearprojection_linear	LayerNormprojection_norm)selfencoder_hid_dimcross_attention_dim	__class__s      r"   r)   Kandinsky3EncoderProj.__init__&   s4    !#?V[!\!||,?@r!   c                 J    U R                  U5      nU R                  U5      nU$ Nr+   r-   )r.   xs     r"   forwardKandinsky3EncoderProj.forward+   s'    ""1%  #r!   r5   r   r   r   r   r)   r7   r    __classcell__r1   s   @r"   r$   r$   %   s    A
 r!   r$   c                      ^  \ rS rSr\        SS\S\S\S\S\\\\   4   S\\   S\\\\   4   S	\4U 4S
 jjj5       r\	S\
\\4   4S j5       rS\\\
\\4   4   4S jrS rSS jrSrU =r$ )Kandinsky3UNet1   in_channelstime_embedding_dimgroupsattention_head_dimlayers_per_blockblock_out_channelsr0   r/   c	                 
  > [         TU ]  5         Sn	Sn
SnSnUnUS   S-  n[        USSS9U l        [	        UU5      U l        [        X'U5      U l        [        R                  " XSSS	9U l
        [        X5      U l        U/[        U5      -   n[        [        US S
 USS  5      5      nU Vs/ s H  nU(       a  UOS PM     nn[        U5      U/-  nUUU/n[!        ["        U5      n/ n[        U5      U l        [        R&                  " / 5      U l        [+        [        U/UQ76 5       Ht  u  nu  u  nnnnnUU R$                  S-
  :g  nUR-                  UU R$                  S-
  :w  a  UOS5        U R(                  R-                  [/        UUUUUUUU	U
UU5      5        Mv     [        R&                  " / 5      U l        [+        [        [#        U5      /UQ76 5       HP  u  nu  u  nnnnnUS:g  nU R0                  R-                  [3        UUR5                  5       UUUUUUU	U
UU5      5        MR     [        R6                  " X>5      U l        [        R:                  " 5       U l        [        R                  " XSSS	9U l        g s  snf )N   r   )FTTTr   F   )flip_sin_to_cosdownscale_freq_shiftr   kernel_sizepadding) r(   r)   r   	time_projr   time_embeddingKandinsky3AttentionPoolingadd_time_conditionr   Conv2dconv_inr$   encoder_hid_projlistziplenmapreversed
num_levels
ModuleListdown_blocks	enumerateappendKandinsky3DownSampleBlock	up_blocksKandinsky3UpSampleBlockpop	GroupNormconv_norm_outSiLUconv_act_outconv_out) r.   r?   r@   rA   rB   rC   rD   r0   r/   expansion_ratiocompression_ratioadd_cross_attentionadd_self_attentionout_channelsinit_channelshidden_dimsin_out_dimsis_exist	text_dims
num_blockslayer_paramsrev_layer_paramscat_dimslevelin_dimout_dimres_block_numtext_dimself_attentiondown_sample	up_sampler1   s                                   r"   r)   Kandinsky3UNet.__init__2   s    	 76"*1-2"=%^_`/

 #=5G#
 yyTUV 5o [$o-?(@@3{3B/QRABObcOb8H($>Ob	c+,0@/AA
"I/ABx6k*==,S\+l+T
OEO%VW}h  DOOa$78KOOu11D'EG1M##)&!&#%"T
* r*S\%9(89T
OEO%Wf}h 
INN!!'LLN&!&#%"	T
*  \\&@GGI		-1VWXk ds   -J returnc                    ^ 0 nS[         S[        R                  R                  S[        [         [
        4   4U4S jjmU R                  5        H  u  p#T" X#U5        M     U$ )z
Returns:
    `dict` of attention processors: A dictionary containing all attention processors used in the model with
    indexed by its weight name.
namemodule
processorsc                    > [        US5      (       a  UR                  X  S3'   UR                  5        H  u  p4T" U  SU 3XB5        M     U$ Nset_processorz
.processor.)hasattr	processornamed_children)r   r   r   sub_namechildfn_recursive_add_processorss        r"   r   CKandinsky3UNet.attn_processors.<locals>.fn_recursive_add_processors   sW    v//282B2B
V:./#)#8#8#:+tfAhZ,@%T $; r!   )strr   r   Moduler   r   r   )r.   r   r   r   r   s       @r"   attn_processorsKandinsky3UNet.attn_processors   sb     
	c 	588?? 	X\]`bt]tXu 	 !//1LD'jA 2 r!   r   c           	      d  ^ [        U R                  R                  5       5      n[        U[        5      (       a-  [        U5      U:w  a  [        S[        U5       SU SU S35      eS[        S[        R                  R                  4U4S jjmU R                  5        H  u  p4T" X4U5        M     g)	a  
Sets the attention processor to use to compute attention.

Parameters:
    processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
        The instantiated processor class or a dictionary of processor classes that will be set as the processor
        for **all** `Attention` layers.

        If `processor` is a dict, the key needs to define the path to the corresponding cross attention
        processor. This is strongly recommended when setting trainable attention processors.

z>A dict of processors was passed, but the number of processors z0 does not match the number of attention layers: z. Please make sure to pass z processor classes.r   r   c                 
  > [        US5      (       aJ  [        U[        5      (       d  UR                  U5        O#UR                  UR	                  U  S35      5        UR                  5        H  u  p4T" U  SU 3XB5        M     g r   )r   
isinstancedictr   rb   r   )r   r   r   r   r   fn_recursive_attn_processors        r"   r   FKandinsky3UNet.set_attn_processor.<locals>.fn_recursive_attn_processor   ss    v//!)T22((3(($z7J)KL#)#8#8#:+tfAhZ,@%S $;r!   N)rW   r   keysr   r   
ValueErrorr   r   r   r   r   )r.   r   countr   r   r   s        @r"   set_attn_processor!Kandinsky3UNet.set_attn_processor   s     D((--/0i&&3y>U+BPQTU^Q_P` a005w6QRWQXXkm 
	Tc 	T588?? 	T !//1LD'i@ 2r!   c                 6    U R                  [        5       5        g)zU
Disables custom attention processors and sets the default attention implementation.
N)r   r   )r.   s    r"   set_default_attn_processor)Kandinsky3UNet.set_default_attn_processor   s     	0r!   c                    Ub2  SUR                  UR                  5      -
  S-  nUR                  S5      n[        R                  " U5      (       dV  [        U[        5      (       a  [        R                  O[        R                  n[        R                  " U/XaR                  S9nO7[        UR                  5      S:X  a  US    R                  UR                  5      nUR                  UR                  S   5      nU R                  U5      R                  UR                  5      nU R                  U5      nU R!                  U5      nUb  U R#                  XU5      n/ n	U R%                  U5      n['        U R(                  5       H3  u  pU" XX45      nXR*                  S-
  :w  d  M"  U	R-                  U5        M5     ['        U R.                  5       H8  u  pU
S:w  a$  [        R0                  " XR3                  5       /SS9nU" XX45      nM:     U R5                  U5      nU R7                  U5      nU R9                  U5      nU(       d  U4$ [;        US9$ )NrG   g     )dtypedevicer   dim)r   )tor   	unsqueezer   	is_tensorr   floatfloat32int32tensorr   rW   shapeexpandrN   rO   rT   rQ   rS   r]   r\   rZ   r^   r`   catrb   rd   rf   rg   r   )r.   r   timestepencoder_hidden_statesencoder_attention_maskreturn_dictr   time_embed_input
time_embedhidden_statesrv   r|   r}   s                r"   r7   Kandinsky3UNet.forward   s   !-&'*@*C*CFLL*Q&QU]%]"%;%E%Ea%H"x((%/%%@%@EMMekkE||XJeMMRH A%~((7H ??6<<?3>>(366v||D(()9:
 $ 5 56K L ,00TjkJf%"+D,<,<"=E 5JcF!++$$V, #>
 !*$.. 9EzF,=,=,?#@aHv3HaF !:
 ##F+""6*v&9#622r!   )rQ   rf   rS   rd   rg   r\   rT   rZ   rO   rN   r`   )rF      r   @   r   )i  i   r   i      r   )NNT)r   r   r   r   r
   intr   r   r)   propertyr   r   r   r   r   r   r7   r    r:   r;   s   @r"   r=   r=   1   s
    "&"$34)?6:#[Y[Y  [Y 	[Y
  [Y  U3Z0[Y "#J[Y #3c
?3[Y [Y [Yz c+=&=!>  . AE2Dd3PbKbFc2c,d  AD1'3 '3r!   r=   c                   F   ^  \ rS rSr        SU 4S jjrSS jrSrU =r$ )ra      c                   > [         TU ]  5         S U(       a  SOS S S //S /S-  /US-
  -  -   nX-   U4/X4/US-
  -  -   X4/-   n/ n/ n/ nXl        XPl        U(       a  UR	                  [        X4S XxU	5      5        O$UR	                  [        R                  " 5       5        [        X5       H  u  u  nnnUR	                  [        UUXGU
U5      5        Ub  UR	                  [        UXEXxU	5      5        O$UR	                  [        R                  " 5       5        UR	                  [        UUXGU
5      5        M     [        R                  " U5      U l        [        R                  " U5      U l        [        R                  " U5      U l        g )NTrF   rG   r   r(   r)   r{   context_dimr^   Kandinsky3AttentionBlockr   IdentityrV   Kandinsky3ResNetBlockr[   
attentions
resnets_inresnets_out)r.   r?   cat_dimrl   time_embed_dimr   rr   rA   head_dimrh   ri   r}   r{   up_resolutionshidden_channelsr   r   r   
in_channelout_channelup_resolutionr1   s                        r"   r)    Kandinsky3UpSampleBlock.__init__   s    	dDIJtfWXj\]gjk]kMll#[12)*j1n=>*+, 	
 

,&(tV_no bkkm,8;O8\4%Z}%j*nVgivw &!!,"NSb !!"++-0%j+~Whi 9]$ --
3--
3==5r!   c                    [        U R                  SS  U R                  U R                  5       H-  u  pgnU" X5      nU R                  b
  U" XX4U5      nU" X5      nM/     U R
                  (       a  U R                  S   " XUS9nU$ )NrG   r   
image_mask)rV   r   r   r   r   r{   	r.   r6   r   contextcontext_maskr   	attention	resnet_in
resnet_outs	            r"   r7   Kandinsky3UpSampleBlock.forward5  s    03DOOAB4GZ^ZjZj0k,I*!(A+aWJO1)A	 1l "1ZHAr!   r   r   r   r   r{   Nr   r   r   rF   r   TTNNNr9   r;   s   @r"   ra   ra      s-     86t	 	r!   ra   c                   F   ^  \ rS rSr        SU 4S jjrSS jrSrU =r$ )r_   iA  c                   > [         TU ]  5         / n/ n/ nXl        X@l        U(       a  UR	                  [        XS XgU5      5        O$UR	                  [        R                  " 5       5        S /S-  /US-
  -  S S U
(       a  SOS S //-   nX4/X"4/US-
  -  -   n[        UU5       H  u  u  nnnUR	                  [        UUX6U	5      5        Ub  UR	                  [        UX4XgU5      5        O$UR	                  [        R                  " 5       5        UR	                  [        UUX6U	U5      5        M     [        R                  " U5      U l        [        R                  " U5      U l        [        R                  " U5      U l        g )NrF   rG   Fr   )r.   r?   rl   r   r   rr   rA   r   rh   ri   r|   r{   r   r   r   r   r   r   r   r   r1   s                       r"   r)   "Kandinsky3DownSampleBlock.__init__B  s~    	

,&(dF^mn bkkm,&1*a8T4R]cgim<n;oo'67L;W:X\fij\j:kk8;O^8\4%Z}%j+~Whi &!!,#^&Tc !!"++-0%nFWYf 9]( --
3--
3==5r!   c                    U R                   (       a  U R                  S   " XUS9n[        U R                  SS  U R                  U R                  5       H-  u  pgnU" X5      nU R
                  b
  U" XX4U5      nU" X5      nM/     U$ )Nr   r   rG   )r{   r   rV   r   r   r   r   s	            r"   r7   !Kandinsky3DownSampleBlock.forwardy  s    "1ZHA03DOOAB4GZ^ZjZj0k,I*!(A+aWJO1)A	 1l
 r!   r   r   r   r9   r;   s   @r"   r_   r_   A  s-     56n	 	r!   r_   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )Kandinsky3ConditionalGroupNormi  c                   > [         TU ]  5         [        R                  " XSS9U l        [        R
                  " [        R                  " 5       [        R                  " USU-  5      5      U l        U R                  S   R                  R                  R                  5         U R                  S   R                  R                  R                  5         g )NF)affiner   rG   )r(   r)   r   rc   norm
Sequentialre   r*   context_mlpweightdatazero_r'   )r.   rA   normalized_shaper   r1   s       r"   r)   'Kandinsky3ConditionalGroupNorm.__init__  s    LL%H	==BIIk1O_K_4`a""''--/  %%++-r!   c                     U R                  U5      n[        [        UR                  SS  5      5       H  nUR	                  S5      nM     UR                  SSS9u  pEU R                  U5      US-   -  U-   nU$ )Nr   rM   rG   r   g      ?)r   rangerW   r   r   chunkr   )r.   r6   r   _scaleshifts         r"   r7   &Kandinsky3ConditionalGroupNorm.forward  sv    ""7+s17712;'(A''+G ) }}QA}.IIaLECK(50r!   )r   r   r9   r;   s   @r"   r   r     s    . r!   r   c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )Kandinsky3Blocki  c                   > [         TU ]  5         [        XQU5      U l        [        R
                  " 5       U l        Ub#  U(       a  [        R                  " XSSS9U l        O[        R                  " 5       U l        [        US:  5      n[        R                  " XXGS9U l        Ub#  U(       d  [        R                  " X"SSS9U l        g [        R                  " 5       U l        g )Nr   rK   striderG   rJ   )r(   r)   r   
group_normr   re   
activationConvTranspose2dr}   r   r   rR   
projectionr|   )	r.   r?   rl   r   rK   norm_groupsr   rL   r1   s	           r"   r)   Kandinsky3Block.__init__  s    8Sab'')$//VW`abDN[[]DNkAo&))K;h$]!yyQR[\]D!{{}Dr!   c                     U R                  X5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU$ r4   )r   r   r}   r   r|   )r.   r6   r   s      r"   r7   Kandinsky3Block.forward  sO    OOA*OOANN1OOAQr!   )r   r|   r   r   r}   )r   r   Nr9   r;   s   @r"   r   r     s    -" r!   r   c                   @   ^  \ rS rSrSSSS/-  4U 4S jjrS rSrU =r$ )	r   i  r   r   rF   Nc                 \  > [         TU ]  5         / SQn[        X5      U-  nX4/X4/S-  -   X4/-   n	[        R                  " [        XU5       V
VVVs/ s H  u  u  pp[        XX<XM5      PM     snnnn
5      U l        SU;   a  [        R                  " XSSS9O[        R                  " 5       U l
        X:w  a  [        R                  " XSS9O[        R                  " 5       U l        SU;   a  [        R                  " X"SSS9U l        g [        R                  " 5       U l        g s  snnnn
f )N)rG   r   r   rG   r   Tr   rG   )rK   F)r(   r)   maxr   r[   rV   r   resnet_blocksr   r   shortcut_up_samplerR   shortcut_projectionshortcut_down_sample)r.   r?   rl   r   r   ri   r   kernel_sizeshidden_channelr   r   r   rK   r   r1   s                 r"   r)   Kandinsky3ResNetBlock.__init__  s;    	#[7;LL*+/O.PST.TTYgXvWww 	  ]] NQ#>NNI-Z{  
VaqN
 ~% {QqQ 	 DOC^BIIkQ?dfdododq 	 
 & IIlaJ 	!  	!s   D&c                     UnU R                    H  nU" X25      nM     U R                  U5      nU R                  U5      nU R                  U5      nX-   nU$ r4   )r  r  r  r	  )r.   r6   r   outresnet_blocks        r"   r7   Kandinsky3ResNetBlock.forward  s_     ..Ls/C / ##A&$$Q'%%a(Gr!   )r  r	  r  r  r9   r;   s   @r"   r   r     s$    EG[\mnrvqwmw
>	 	r!   r   c                   6   ^  \ rS rSrSU 4S jjrSS jrSrU =r$ )rP   i  c                 F   > [         TU ]  5         [        UUUUSS9U l        g )NFdim_headrx   out_bias)r(   r)   r   r   )r.   num_channelsr   r   r1   s       r"   r)   #Kandinsky3AttentionPooling.__init__  s)    " 
r!   c                     UR                  UR                  S9nU R                  UR                  SSS9X#5      nXR	                  S5      -   $ )Nr   rG   T)r   keepdim)r   r   r   meansqueeze)r.   r6   r   r   s       r"   r7   "Kandinsky3AttentionPooling.forward  sE    #W]];..!T!BGZ??1%%%r!   )r   )r   r4   r9   r;   s   @r"   rP   rP     s    
& &r!   rP   c                   6   ^  \ rS rSrSU 4S jjrSS jrSrU =r$ )r   i  c                 T  > [         TU ]  5         [        XAU5      U l        [	        UU=(       d    UUUSS9U l        Xa-  n[        XAU5      U l        [        R                  " [        R                  " XSSS9[        R                  " 5       [        R                  " XqSSS95      U l        g )NFr  rG   )rK   r'   )r(   r)   r   in_normr   r   out_normr   r   rR   re   feed_forward)	r.   r  r   r   r   r   rh   r   r1   s	           r"   r)   !Kandinsky3AttentionBlock.__init__  s    5kQ_`"'< 
 *86{R`aMMIIlOGGIIIoO
r!   c                    UR                   SS  u  pgU R                  X5      nUR                  UR                   S   SXg-  5      R                  SSS5      nUb  UOUnUb  UR	                  UR
                  S9nU R                  XU5      nUR                  SSS5      R                  S5      R                  UR                   S   SXg5      nX-   nU R                  X5      nU R                  U5      nX-   nU$ )Nr   rM   r   rG   r  )
r   r   reshapepermuter   r   r   r   r!  r"  )	r.   r6   r   r   r   r   heightwidthr  s	            r"   r7    Kandinsky3AttentionBlock.forward  s    ll1)kk!''!*b&.9AA!QJ$0'c#'????LnnS<8kk!Q",,R0881r6YGmmA*$Gr!   )r   r"  r   r!  )Nr   r   rF   r   r9   r;   s   @r"   r   r     s    
& r!   r   )&dataclassesr   typingr   r   r   r   torch.utils.checkpointr   configuration_utilsr	   r
   utilsr   r   attention_processorr   r   r   
embeddingsr   r   modeling_utilsr   
get_loggerr   loggerr   r   r$   r=   ra   r_   r   r   r   rP   r   r   r!   r"   <module>r5     s    " % %    B ( N N 5 ' 
		H	%  :    	BII 	F3Z F3RDbii DNA		 AHRYY &bii 6)BII )X& &"#ryy #r!   