
    +hK                    .   S SK r S SKrS SKJrJrJrJrJr  S SKrS SK	J
s  Jr  S SKJ
r
  SSKJr  SSKJrJrJr  SSKJrJrJr  SSKJrJr  \R4                  " \5      r\" 5       (       a  S SKr\" 5       (       a	  S SKrS SKrOSr\" 5       (       a  \" S	S
5      (       a  S SK J!r!  S SK"J#r#  Sr$OSr$\ " S S\
RJ                  5      5       r& " S S\
RJ                  5      r' " S S\
RJ                  5      r( " S S\
RJ                  5      r) " S S5      r* " S S5      r+ " S S\
RJ                  5      r, " S S5      r- " S S 5      r. " S! S"5      r/ " S# S$5      r0 " S% S&5      r1 " S' S(5      r2 " S) S*5      r3 " S+ S,5      r4 " S- S.5      r5 " S/ S05      r6 " S1 S25      r7 " S3 S45      r8 " S5 S65      r9 " S7 S85      r: " S9 S:5      r; " S; S<5      r< " S= S>5      r= " S? S@5      r> " SA SB5      r? " SC SD5      r@ " SE SF5      rA " SG SH5      rB " SI SJ5      rC " SK SL5      rD " SM SN5      rE " SO SP\
RJ                  5      rF " SQ SR\
RJ                  5      rG " SS ST5      rH " SU SV5      rI " SW SX\
RJ                  5      rJ " SY SZ\
RJ                  5      rK " S[ S\\R                  RJ                  5      rL " S] S^\R                  RJ                  5      rM " S_ S`\R                  RJ                  5      rN " Sa Sb5      rO " Sc Sd5      rP " Se Sf5      rQ " Sg Sh5      rR " Si Sj5      rS " Sk Sl5      rT " Sm Sn5      rU " So Sp5      rV " Sq Sr5      rW " Ss St5      rX " Su Sv5      rY " Sw Sx5      rZ " Sy Sz5      r[ " S{ S|5      r\ " S} S~5      r] " S S5      r^ " S S5      r_\-\I\.\94r`\+\<\:\H\K\L\\4ra\/ \+P\,P\-P\.P\/P\0P\1P\2P\4P\5P\6P\YP\]P\[P\^P\7P\8P\9P\:P\=P\;P\<P\>P\*P\?P\@P\AP\BP\CP\DP\EP\FP\GP\HP\IP\VP\WP\XP\(P\QP\'P\KP\LP\MP\NP\OP\PP\RP\SP\TP\UP7   rbg)    N)CallableListOptionalTupleUnion)nn   )IPAdapterMaskProcessor)	deprecateis_torch_xla_availablelogging)is_torch_npu_availableis_torch_xla_versionis_xformers_available)is_torch_versionmaybe_allow_in_graph>z2.2)flash_attention)is_spmdTFc            9         ^  \ rS rSrSr                             SBS\S\\   S\S\\   S\S	\S
\S\S\S\\	   S\S\\	   S\\   S\\   S\\   S\\   S\S\S\S\S\S\S\S\S   S\S\S\S \48U 4S! jjjr
  SCS"\S#\\\\	   S$4      S%S4S& jjrS'\S%S4S( jr SDS)\S*\\   S%S4S+ jjrS,\S%S4S- jrSES. jrSFS/\S%S04S1 jjr  SGS2\R(                  S3\\R(                     S4\\R(                     S%\R(                  4S5 jjrS6\R(                  S%\R(                  4S7 jrSHS6\R(                  S\S%\R(                  4S8 jjr SDS9\R(                  S:\R(                  S4\\R(                     S%\R(                  4S; jjr SHS4\R(                  S<\S=\S\S%\R(                  4
S> jjrS3\R(                  S%\R(                  4S? jr\R6                  " 5       SIS@ j5       rSArU =r$ )J	Attention1   a  
A cross attention layer.

Parameters:
    query_dim (`int`):
        The number of channels in the query.
    cross_attention_dim (`int`, *optional*):
        The number of channels in the encoder_hidden_states. If not given, defaults to `query_dim`.
    heads (`int`,  *optional*, defaults to 8):
        The number of heads to use for multi-head attention.
    kv_heads (`int`,  *optional*, defaults to `None`):
        The number of key and value heads to use for multi-head attention. Defaults to `heads`. If
        `kv_heads=heads`, the model will use Multi Head Attention (MHA), if `kv_heads=1` the model will use Multi
        Query Attention (MQA) otherwise GQA is used.
    dim_head (`int`,  *optional*, defaults to 64):
        The number of channels in each head.
    dropout (`float`, *optional*, defaults to 0.0):
        The dropout probability to use.
    bias (`bool`, *optional*, defaults to False):
        Set to `True` for the query, key, and value linear layers to contain a bias parameter.
    upcast_attention (`bool`, *optional*, defaults to False):
        Set to `True` to upcast the attention computation to `float32`.
    upcast_softmax (`bool`, *optional*, defaults to False):
        Set to `True` to upcast the softmax computation to `float32`.
    cross_attention_norm (`str`, *optional*, defaults to `None`):
        The type of normalization to use for the cross attention. Can be `None`, `layer_norm`, or `group_norm`.
    cross_attention_norm_num_groups (`int`, *optional*, defaults to 32):
        The number of groups to use for the group norm in the cross attention.
    added_kv_proj_dim (`int`, *optional*, defaults to `None`):
        The number of channels to use for the added key and value projections. If `None`, no projection is used.
    norm_num_groups (`int`, *optional*, defaults to `None`):
        The number of groups to use for the group norm in the attention.
    spatial_norm_dim (`int`, *optional*, defaults to `None`):
        The number of channels to use for the spatial normalization.
    out_bias (`bool`, *optional*, defaults to `True`):
        Set to `True` to use a bias in the output linear layer.
    scale_qk (`bool`, *optional*, defaults to `True`):
        Set to `True` to scale the query and key by `1 / sqrt(dim_head)`.
    only_cross_attention (`bool`, *optional*, defaults to `False`):
        Set to `True` to only use cross attention and not added_kv_proj_dim. Can only be set to `True` if
        `added_kv_proj_dim` is not `None`.
    eps (`float`, *optional*, defaults to 1e-5):
        An additional value added to the denominator in group normalization that is used for numerical stability.
    rescale_output_factor (`float`, *optional*, defaults to 1.0):
        A factor to rescale the output by dividing it with this value.
    residual_connection (`bool`, *optional*, defaults to `False`):
        Set to `True` to add the residual connection to the output.
    _from_deprecated_attn_block (`bool`, *optional*, defaults to `False`):
        Set to `True` if the attention block is loaded from a deprecated state dict.
    processor (`AttnProcessor`, *optional*, defaults to `None`):
        The attention processor to use. If `None`, defaults to `AttnProcessor2_0` if `torch 2.x` is used and
        `AttnProcessor` otherwise.
N	query_dimcross_attention_dimheadskv_headsdim_headdropoutbiasupcast_attentionupcast_softmaxcross_attention_normcross_attention_norm_num_groupsqk_normadded_kv_proj_dimadded_proj_biasnorm_num_groupsspatial_norm_dimout_biasscale_qkonly_cross_attentionepsrescale_output_factorresidual_connection_from_deprecated_attn_block	processorAttnProcessorout_dimout_context_dimelementwise_affine	is_causalc                   > [         T#U ]  5         SSKJnJn Jn!  Ub  UOXS-  U l        Uc  U R                  OXT-  U l        Xl        Xpl	        US LU l
        Ub  UOUU l        Xl        Xl        UU l        UU l        X`l        SU l        Ub  UOUU l        Ub  UOUU l        UU l        UU l        UU l        UU l        UU l        U R0                  (       a  US-  OSU l        Ub  UU-  OUU l        X0l        Xl        UU l        U R8                  c  U R:                  (       a  [=        S5      eUb  [>        R@                  " XUSS9U l!        OS U l!        Ub  [E        UUS	9U l#        OS U l#        Uc  S U l$        S U l%        GOUS
:X  a7  [>        RL                  " UUUS9U l$        [>        RL                  " UUUS9U l%        OUS:X  a  U" USSUS9U l$        U" USSUS9U l%        OUS:X  a9  [>        RL                  " XS-  US9U l$        [>        RL                  " XT-  US9U l%        OtUS:X  a  U!" UUUS9U l$        U!" UUUS9U l%        OSUS:X  a  U!" XS-  US9U l$        U!" XT-  US9U l%        O0US:X  a  U " SSUS9U l$        U " SSUS9U l%        O[=        SU S35      eU
c  S U l'        OzU
S
:X  a&  [>        RL                  " U R                  5      U l'        ONU
S:X  a9  U R8                  b  Un"OU R                  n"[>        R@                  " U"USSS9U l'        O[=        SU
 S35      e[>        RP                  " XR                  US9U l)        U R:                  (       d_  [>        RP                  " U R                  U R                  US9U l*        [>        RP                  " U R                  U R                  US9U l+        OS U l*        S U l+        Xl,        U R8                  bz  [>        RP                  " XR                  US9U l-        [>        RP                  " XR                  US9U l.        U R(                  b$  [>        RP                  " XR                  US9U l/        OS U l/        S U l-        S U l.        U R*                  (       d  [>        R`                  " / 5      U l1        U Rb                  Re                  [>        RP                  " U R                  U R$                  US95        U Rb                  Re                  [>        Rf                  " U5      5        OS U l1        U R(                  bA  U R(                  (       d0  [>        RP                  " U R                  U R&                  US9U l4        OS U l4        Ub  Ub  US
:X  a7  [>        RL                  " UUUS9U l5        [>        RL                  " UUUS9U l6        O{US:X  a  U" USSUS9U l5        U" USSUS9U l6        OXUS:X  a  U!" UUS9U l5        U!" UUS9U l6        O9US:X  a  S U l5        U!" XT-  US9U l6        O[=        SU S35      eS U l5        S U l6        Uc:  [o        [p        S5      (       a  U R0                  (       a
  [s        5       O	[u        5       nU Rw                  U5        g )N   )FP32LayerNormLpNormRMSNormFg            ?z`only_cross_attention` can only be set to True if `added_kv_proj_dim` is not None. Make sure to set either `only_cross_attention=False` or define `added_kv_proj_dim`.Tnum_channels
num_groupsr,   affine)
f_channelszq_channels
layer_norm)r,   r4   fp32_layer_norm)r4   r   r,   layer_norm_across_heads)r,   rms_normrms_norm_across_headsl2r	   )pdimr,   zunknown qk_norm: z. Should be one of None, 'layer_norm', 'fp32_layer_norm', 'layer_norm_across_heads', 'rms_norm', 'rms_norm_across_heads', 'l2'.
group_normh㈵>zunknown cross_attention_norm: z.. Should be None, 'layer_norm' or 'group_norm'r   zC. Should be one of `None,'layer_norm','fp32_layer_norm','rms_norm'`scaled_dot_product_attention)<super__init__normalizationr8   r9   r:   	inner_diminner_kv_dimr   use_biasis_cross_attentionr   r    r!   r-   r.   r   fused_projectionsr2   r3   context_pre_onlypre_onlyr5   r/   r*   scaler   sliceable_head_dimr%   r+   
ValueErrorr   	GroupNormrK   SpatialNormspatial_normnorm_qnorm_k	LayerNorm
norm_crossLinearto_qto_kto_vr&   
add_k_proj
add_v_proj
add_q_proj
ModuleListto_outappendDropout
to_add_outnorm_added_qnorm_added_khasattrFAttnProcessor2_0r1   set_processor)$selfr   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r2   r3   rW   rX   r4   r5   r8   r9   r:   norm_cross_num_channels	__class__s$                                      ^/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/models/attention_processor.pyrP   Attention.__init__i   s   B 	 	BA$+$7X=M.6.>DNNHDW""5T"A:M:Y#6_h  0,%:"#6 !&")"5w92A2MS\ 0 " ,G( '+}}Xt^#
,3,?W(U
 #(!2$8!!!)d.G.G y  & ll	cfostDO"DO' +yN^ _D $D?DKDK$,,xSM_`DK,,xSM_`DK))'UQV\_`DK'UQV\_`DK11,,x'7SADK,,x':DDK
"!(HZ[DK!(HZ[DK//!("2<DK!("53?DK_ 1"#6DK 1"#6DK#G9  -l  m   '"DO!\1 ll4+C+CDDO!\1%%1 +<'*.*B*B' ll4A`fjswDO 01E0FFtu  IIidC	((		$":":D<M<MTXYDI		$":":D<M<MTXYDIDIDI.!!- ii(9;L;LSbcDO ii(9;L;LSbcDO$$0"$)),=~~Tc"d"DO"DO"DO}}--+DKKKryyHUVKKrzz'23DK  ,T5J5J ii8L8LS[\DO"DO#4#@,&$&LLsWi$j!$&LLsWi$j!--$1(u[`fi$j!$1(u[`fi$j!J&$+H#$>!$+H#$>!33 %)!$+H,?S$I! 'y0st  !%D $D &-a1O&P&PUYUbUb "huhw  	9%    use_xla_flash_attentionpartition_spec.returnc                 j   U(       aa  [         (       d  Se[        SS5      (       a  Se[        5       (       a  [        SS5      (       a  SeU(       a  [        U5      nOF[	        U5      nO:[        [        S5      (       a  U R                  (       a
  [        5       O	[        5       nU R                  U5        g)	a3  
Set whether to use xla flash attention from `torch_xla` or not.

Args:
    use_xla_flash_attention (`bool`):
        Whether to use pallas flash attention kernel from `torch_xla` or not.
    partition_spec (`Tuple[]`, *optional*):
        Specify the partition specification if using SPMD. Otherwise None.
ztorch_xla is not available<2.3zEflash attention pallas kernel is supported from torch_xla version 2.32.4zPflash attention pallas kernel using SPMD is supported from torch_xla version 2.4rN   N)r   r   r   XLAFluxFlashAttnProcessor2_0XLAFlashAttnProcessor2_0rq   rr   r*   rs   r1   rt   )ru   r{   r|   is_fluxr0   s        rx   set_use_xla_flash_attention%Attention.set_use_xla_flash_attention5  s     #))22%c511]]3C??hh <^ LI 8 HI '.a1O&P&PUYUbUb "huhw  	9%rz   use_npu_flash_attentionc                     U(       a  [        5       nO:[        [        S5      (       a  U R                  (       a
  [	        5       O	[        5       nU R                  U5        g)zB
Set whether to use npu flash attention from `torch_npu` or not.

rN   N)AttnProcessorNPUrq   rr   r*   rs   r1   rt   )ru   r   r0   s      rx   set_use_npu_flash_attention%Attention.set_use_npu_flash_attentionV  sI    
 #(*I '.a1O&P&PUYUbUb "huhw  	9%rz   'use_memory_efficient_attention_xformersattention_opc                    [        U S5      =(       a%    [        U R                  [        [        [
        45      n[        U S5      =(       a*    [        U R                  [        [        [        [        45      n[        U S5      =(       a%    [        U R                  [        [        [        45      n[        U S5      =(       a     [        U R                  [        [        45      nU(       Ga  U(       a  U(       a  [        SU R                   35      e[!        5       (       d
  [#        SSS9e[$        R&                  R)                  5       (       d  [+        S5      e SnUb  Uu  pUR,                  tpz[$        R.                  " SS	US
9n[0        R2                  R5                  XU5      n
U(       a  [	        U R                  R8                  U R                  R:                  U R                  R<                  U R                  R>                  US9nURA                  U R                  RC                  5       5        [        U R                  S5      (       a9  URE                  U R                  RF                  RH                  RJ                  5        GOU(       a   [L        RO                  S5        [        US9nGOwU(       Ga	  [        U R                  R<                  U R                  R>                  U R                  RP                  U R                  RR                  US9nURA                  U R                  RC                  5       5        [        U R                  S5      (       af  URE                  U R                  RT                  S   RH                  RJ                  U R                  RT                  S   RH                  RV                  S
9  GOfU(       a  [        US9nGOT[Y        US9nGOIU(       a  [        [Z        S5      (       a  [
        O[        nU" U R                  R8                  U R                  R:                  U R                  R<                  U R                  R>                  S9nURA                  U R                  RC                  5       5        [        U R                  S5      (       a9  URE                  U R                  RF                  RH                  RJ                  5        GOIU(       Ga  [        U R                  R<                  U R                  R>                  U R                  RP                  U R                  RR                  S9nURA                  U R                  RC                  5       5        [        U R                  S5      (       af  URE                  U R                  RT                  S   RH                  RJ                  U R                  RT                  S   RH                  RV                  S
9  O:[        [Z        S5      (       a  U R\                  (       a
  [_        5       O	[a        5       nU Rc                  U5        g! [6         a  nUeSnAff = f)ax  
Set whether to use memory efficient attention from `xformers` or not.

Args:
    use_memory_efficient_attention_xformers (`bool`):
        Whether to use memory efficient attention from `xformers` or not.
    attention_op (`Callable`, *optional*):
        The attention operation to use. Defaults to `None` which uses the default attention operation from
        `xformers`.
r0   zhMemory efficient attention is currently not supported for custom diffusion for attention processor type zeRefer to https://github.com/facebookresearch/xformers for more information on how to install xformersxformers)namezvtorch.cuda.is_available() should be True but is False. xformers' memory efficient attention is only available for GPU N)r7   r	   (   cudadevicedtype)train_kvtrain_q_outhidden_sizer   r   to_k_custom_diffusionzMemory efficient attention with `xformers` might currently not work correctly if an attention mask is required for the attention operation.r   )r   r   
num_tokensrY   r   to_k_ipr   rN   )r   r   r   r   )r   r   r   rY   )2rq   
isinstancer0   CustomDiffusionAttnProcessor$CustomDiffusionXFormersAttnProcessorCustomDiffusionAttnProcessor2_0AttnAddedKVProcessorAttnAddedKVProcessor2_0SlicedAttnAddedKVProcessorXFormersAttnAddedKVProcessorIPAdapterAttnProcessorIPAdapterAttnProcessor2_0IPAdapterXFormersAttnProcessorJointAttnProcessor2_0XFormersJointAttnProcessorNotImplementedErrorr   ModuleNotFoundErrortorchr   is_availabler[   SUPPORTED_DTYPESrandnr   opsmemory_efficient_attention	Exceptionr   r   r   r   load_state_dict
state_dicttor   weightr   loggerinfor   rY   r   r   XFormersAttnProcessorrr   r*   rs   r1   rt   )ru   r   r   is_custom_diffusionis_added_kv_processoris_ip_adapteris_joint_processorr   op_fwop_bw_qer0   attn_processor_classs                  rx   +set_use_memory_efficient_attention_xformers5Attention.set_use_memory_efficient_attention_xformersg  s    &dK8 
ZNN)+OQpq>
 !(k : !
zNN$'*,	@
  k2 
zNN#%>@^_8
 %T;7 
JNN%*=
 3$)<)~  @D  @N  @N  O  P  )**)$ $  ZZ,,.. / 
	 E#/'3$)$:$:	JvUKA ??aHA #@!^^44 $ : : $ : :(,(J(J!-	 ))$..*C*C*EF4>>+BCCLL!E!E!L!L!S!ST&
  b 9lS	: $ : :(,(J(J#~~88....!-	 ))$..*C*C*EF4>>955LL#~~55a8??FFdnnNdNdefNgNnNnNtNt !  $6LQ	1|L	" q"@AA 45 %
 1!^^44 $ : : $ : :(,(J(J		 ))$..*C*C*EF4>>+BCCLL!E!E!L!L!S!ST5 $ : :(,(J(J#~~88....		 ))$..*C*C*EF4>>955LL#~~55a8??FFdnnNdNdefNgNnNnNtNt !  q"@AAdmm %&&  	9%_ ! Gs   AX6 6
Y YY
slice_sizec                 |   Ub+  XR                   :  a  [        SU SU R                    S35      eUb  U R                  b  [        U5      nOaUb  [	        U5      nORU R                  b  [        5       nO:[        [        S5      (       a  U R                  (       a
  [        5       O	[        5       nU R                  U5        g)z
Set the slice size for attention computation.

Args:
    slice_size (`int`):
        The slice size for attention computation.
Nzslice_size z has to be smaller or equal to .rN   )rZ   r[   r%   r   SlicedAttnProcessorr   rq   rr   r*   rs   r1   rt   )ru   r   r0   s      rx   set_attention_sliceAttention.set_attention_slice  s     !j3J3J&J{:,6UVZVmVmUnnopqq!d&<&<&H2:>I#+J7I##/,.I '.a1O&P&PUYUbUb "huhw  	9%rz   c                 j   [        U S5      (       a  [        U R                  [        R                  R
                  5      (       ai  [        U[        R                  R
                  5      (       d@  [        R                  SU R                   SU 35        U R                  R                  S5        Xl        g)zu
Set the attention processor to use.

Args:
    processor (`AttnProcessor`):
        The attention processor to use.
r0   z-You are removing possibly trained weights of z with N)
rq   r   r0   r   r   Moduler   r   _modulespop)ru   r0   s     rx   rt   Attention.set_processor  sx     D+&&4>>588??;;y%((//::KKGGWW]^g]hijMMk*"rz   return_deprecated_loraAttentionProcessorc                 *    U(       d  U R                   $ g)z
Get the attention processor in use.

Args:
    return_deprecated_lora (`bool`, *optional*, defaults to `False`):
        Set to `True` to return the deprecated LoRA attention processor.

Returns:
    "AttentionProcessor": The attention processor in use.
Nr0   )ru   r   s     rx   get_processorAttention.get_processor)  s     &>>! &rz   hidden_statesencoder_hidden_statesattention_maskc                 2   [        [        R                  " U R                  R                  5      R
                  R                  5       5      nSS1nUR                  5        VVs/ s H  u  pxXu;  d  M  Xv;  d  M  UPM     n	nn[        U	5      S:  a:  [        R                  SU	 SU R                  R                  R                   S35        UR                  5        VV
s0 s H  u  pzXu;   d  M  Xz_M     nnn
U R                  " U U4UUS.UD6$ s  snnf s  sn
nf )a   
The forward method of the `Attention` class.

Args:
    hidden_states (`torch.Tensor`):
        The hidden states of the query.
    encoder_hidden_states (`torch.Tensor`, *optional*):
        The hidden states of the encoder.
    attention_mask (`torch.Tensor`, *optional*):
        The attention mask to use. If `None`, no mask is applied.
    **cross_attention_kwargs:
        Additional keyword arguments to pass along to the cross attention.

Returns:
    `torch.Tensor`: The output of the attention layer.
ip_adapter_masksip_hidden_statesr   zcross_attention_kwargs z are not expected by z and will be ignored.r   r   )setinspect	signaturer0   __call__
parameterskeysitemslenr   warningrw   __name__)ru   r   r   r   cross_attention_kwargsattn_parametersquiet_attn_parameterskr   unused_kwargsws              rx   forwardAttention.forward7  s"   6 g//0G0GHSSXXZ[!35G H0668
8$!A<TAYZYwA8 	 
 }!NN)-8MdnnNfNfNoNoMp  qF  G 4J3O3O3Q!j3Q41UVUi$!$3Q!j~~
 #8)	

 %
 	

 "ks   #D2D9DD-Dtensorc                     U R                   nUR                  u  p4nUR                  X2-  X$U5      nUR                  SSSS5      R                  X2-  XEU-  5      nU$ )a+  
Reshape the tensor from `[batch_size, seq_len, dim]` to `[batch_size // heads, seq_len, dim * heads]`. `heads`
is the number of heads initialized while constructing the `Attention` class.

Args:
    tensor (`torch.Tensor`): The tensor to reshape.

Returns:
    `torch.Tensor`: The reshaped tensor.
r   r	   r7      )r   shapereshapepermute)ru   r   	head_size
batch_sizeseq_lenrJ   s         rx   batch_to_head_dimAttention.batch_to_head_dime  s_     JJ	#)<< 
S
 7SQ1a+33J4KW\eVefrz   c                    U R                   nUR                  S:X  a  UR                  u  pEnSnOUR                  u  pGpVUR                  XEU-  X6U-  5      nUR	                  SSSS5      nUS:X  a  UR                  XC-  XW-  Xc-  5      nU$ )a  
Reshape the tensor from `[batch_size, seq_len, dim]` to `[batch_size, seq_len, heads, dim // heads]` `heads` is
the number of heads initialized while constructing the `Attention` class.

Args:
    tensor (`torch.Tensor`): The tensor to reshape.
    out_dim (`int`, *optional*, defaults to `3`): The output dimension of the tensor. If `3`, the tensor is
        reshaped to `[batch_size * heads, seq_len, dim // heads]`.

Returns:
    `torch.Tensor`: The reshaped tensor.
r   r7   r   r	   )r   ndimr   r   r   )ru   r   r2   r   r   r   rJ   	extra_dims           rx   head_to_batch_dimAttention.head_to_batch_dimv  s     JJ	;;!'-||$JI28,,/J7
i,?S\L\]1a+a<^^J$:G<OQTQabFrz   querykeyc                 $   UR                   nU R                  (       a   UR                  5       nUR                  5       nUcV  [        R                  " UR
                  S   UR
                  S   UR
                  S   UR                   UR                  S9nSnOUnSn[        R                  " UUUR                  SS5      UU R                  S9nAU R                  (       a  UR                  5       nUR                  SS9nAUR                  U5      nU$ )a+  
Compute the attention scores.

Args:
    query (`torch.Tensor`): The query tensor.
    key (`torch.Tensor`): The key tensor.
    attention_mask (`torch.Tensor`, *optional*): The attention mask to use. If `None`, no mask is applied.

Returns:
    `torch.Tensor`: The attention probabilities/scores.
r   r7   r   r   rH   )betaalpharJ   )r   r    floatr   emptyr   r   baddbmm	transposerY   r!   softmaxr   )	ru   r   r  r   r   baddbmm_inputr  attention_scoresattention_probss	            rx   get_attention_scoresAttention.get_attention_scores  s       KKME))+C!!KKAA		!EKKX]XdXdM D*MD ==MM"b!**
 /557*22r2:),,U3rz   target_lengthr   c                 ^   U R                   nUc  U$ UR                  S   nXb:w  a  UR                  R                  S:X  a_  UR                  S   UR                  S   U4n[        R
                  " XqR                  UR                  S9n[        R                  " X/SS9nO[        R                  " USU4SS	9nUS
:X  a8  UR                  S   X5-  :  a!  UR                  USUR                  S   U-  S9nU$ US:X  a2  UR                  S5      nUR                  USUR                  S   U-  S9nU$ )a)  
Prepare the attention mask for the attention computation.

Args:
    attention_mask (`torch.Tensor`):
        The attention mask to prepare.
    target_length (`int`):
        The target length of the attention mask. This is the length of the attention mask after padding.
    batch_size (`int`):
        The batch size, which is used to repeat the attention mask.
    out_dim (`int`, *optional*, defaults to `3`):
        The output dimension of the attention mask. Can be either `3` or `4`.

Returns:
    `torch.Tensor`: The prepared attention mask.
rH   mpsr   r7   r  r	   r          )valuer   rJ   output_size   )r   r   r   typer   zerosr   catrr   padrepeat_interleave	unsqueeze)	ru   r   r  r   r2   r   current_lengthpadding_shapepaddings	            rx   prepare_attention_mask Attention.prepare_attention_mask  sQ   & JJ	!!!,2226*$$))U2 "0!5!5a!8.:N:Nq:QS` a++m;O;OXfXmXmn!&N+D!!L "#~=7IQT!Ua<##A&)??!/!A!A1.2F2Fq2II2U "B "  \+55a8N+==qn.B.B1.E	.Q > N rz   c                 f   U R                   c   S5       e[        U R                   [        R                  5      (       a  U R                  U5      nU$ [        U R                   [        R                  5      (       a7  UR                  SS5      nU R                  U5      nUR                  SS5      nU$  e)a  
Normalize the encoder hidden states. Requires `self.norm_cross` to be specified when constructing the
`Attention` class.

Args:
    encoder_hidden_states (`torch.Tensor`): Hidden states of the encoder.

Returns:
    `torch.Tensor`: The normalized encoder hidden states.
zGself.norm_cross must be defined to call self.norm_encoder_hidden_statesr7   r	   )rb   r   r   ra   r\   r  )ru   r   s     rx   norm_encoder_hidden_states$Attention.norm_encoder_hidden_states  s     *u,uu*door||44$(OO4I$J! %$ 66 %:$C$CAq$I!$(OO4I$J!$9$C$CAq$I! %$ 5rz   c                 	   U R                   R                  R                  R                  nU R                   R                  R                  R                  nU R
                  (       Gd  [        R                  " U R                   R                  R                  U R                  R                  R                  U R                  R                  R                  /5      nUR                  S   nUR                  S   n[        R                  " XVU R                  X#S9U l        U R                  R                  R                  U5        U R                  (       a  [        R                  " U R                   R                   R                  U R                  R                   R                  U R                  R                   R                  /5      nU R                  R                   R                  U5        GOF[        R                  " U R                  R                  R                  U R                  R                  R                  /5      nUR                  S   nUR                  S   n[        R                  " XVU R                  X#S9U l        U R"                  R                  R                  U5        U R                  (       ay  [        R                  " U R                  R                   R                  U R                  R                   R                  /5      nU R"                  R                   R                  U5        [%        U SS 5      Gb  [%        U SS 5      Gb  [%        U SS 5      Gb  [        R                  " U R&                  R                  R                  U R(                  R                  R                  U R*                  R                  R                  /5      nUR                  S   nUR                  S   n[        R                  " XVU R,                  X#S9U l        U R.                  R                  R                  U5        U R,                  (       a  [        R                  " U R&                  R                   R                  U R(                  R                   R                  U R*                  R                   R                  /5      nU R.                  R                   R                  U5        Xl        g )Nr7   r   )r   r   r   ri   rg   rh   )rd   r   datar   r   rU   r   r  re   rf   r   r   rc   rT   to_qkvcopy_r   to_kvgetattrri   rg   rh   r&   to_added_qkvrV   )ru   fuser   r   concatenated_weightsin_featuresout_featuresconcatenated_biass           rx   fuse_projectionsAttention.fuse_projections  sg   !!&&--		  %%++&&&#(99dii.>.>.C.CTYYEUEUEZEZ\`\e\e\l\l\q\q-r#s .44Q7K/55a8L ))KDMMZ`nDKKK$$%9:}}$)IItyy~~/B/BDIINNDWDWY]YbYbYgYgYlYl.m$n!  &&'89 $)99dii.>.>.C.CTYYEUEUEZEZ-[#\ .44Q7K/55a8L;4==Y_mDJJJ##$89}}$)IItyy~~/B/BDIINNDWDW.X$Y!

%%&78 D,-9lD1=lD1=#(99'',,doo.D.D.I.I4??KaKaKfKfg$  /44Q7K/55a8L "		0D0DV!D $$**+?@##$)II__))..0D0D0I0I4??K_K_KdKde%! !!&&,,->?!%rz   ),r/   rg   ri   rh   r%   r&   rW   r   r   rV   rK   r   rR   rS   r5   rU   rp   ro   rb   r`   r_   r+   r3   r2   rX   r0   r   r-   r.   rY   r*   rZ   r^   rn   r.  re   r,  rk   rd   r*  rf   r    r!   rT   )N   N@   r  FFFN    NNTNNTTFrL   r;   FFNNNNFTF)NFN)r0   r1   r}   N)FNN)r   )T)r   
__module____qualname____firstlineno____doc__intr   r  boolstrrP   r   r   r   r   r   r   rt   r   r   Tensorr   r   r   r  r#  r&  no_gradr4  __static_attributes____classcell__rw   s   @rx   r   r   1   s   4r .2"&!&$.2/1!%+/*.)-*.%*'*$),1/3##'?J&J& &c]J& 	J&
 3-J& J& J& J& J& J& 'smJ& *-J& #J& $C=J& "$J&  "#!J&" #3-#J&$ %J&& 'J&( #)J&* +J&,  %-J&. "/J&0 &*1J&2 O,3J&4 5J&6 7J&< !=J&> ?J& J&^ ?C	&!%& !x}c'9!:;&
 
&B&4 &D &$ aeP&7;P&KST\K]P&	P&d&c &d &8#("D "EY "" 9=15	,
||,
  (5,
 !.	,
 
,
\  " s 5<< 8 `d-\\-(--FNu||F\-	-` ab1#ll1;>1LO1Z]1	1f% %QVQ]Q] %: ]]_2& 2&rz   r   c                   v   ^  \ rS rSrS\S\S\SS4U 4S jjrS\R                  S\R                  4S	 jrS
r	U =r
$ )!SanaMultiscaleAttentionProjectioniF  in_channelsnum_attention_headskernel_sizer}   Nc           
         > [         TU ]  5         SU-  n[        R                  " UUUUS-  USS9U l        [        R                  " XDSSSSU-  SS9U l        g )Nr   r	   F)r"  groupsr   r7   r   )rM  r   )rO   rP   r   Conv2dproj_inproj_out)ru   rI  rJ  rK  channelsrw   s        rx   rP   *SanaMultiscaleAttentionProjection.__init__G  sc     	{?yy1$
 		(aAaJ]F]dijrz   r   c                 J    U R                  U5      nU R                  U5      nU$ r9  rO  rP  ru   r   s     rx   r   )SanaMultiscaleAttentionProjection.forwardZ  s$    ]3m4rz   rT  )r   r;  r<  r=  r?  rP   r   rB  r   rD  rE  rF  s   @rx   rH  rH  F  sS    kk !k 	k
 
k&U\\ ell  rz   rH  c                     ^  \ rS rSrSr       SS\S\S\\   S\S\S\S	\	\S
4   S\S\
4U 4S jjjrS\R                  S\R                  S\R                  S\R                  4S jrS\R                  S\R                  S\R                  S\R                  4S jrS\R                  S\R                  4S jrSrU =r$ )SanaMultiscaleLinearAttentioni`  z(Lightweight multi-scale linear attentionrI  out_channelsrJ  attention_head_dimmult	norm_typekernel_sizes.r,   r.   c
                 x  > [         TU ]  5         SSKJn
  Xl        X@l        X`l        Xl        Uc  [        X-  U-  5      OUnX4-  n[        R                  " XSS9U l        [        R                  " XSS9U l        [        R                  " XSS9U l        [        R                  " 5       U l        U H(  nU R                  R!                  [#        XU5      5        M*     [        R$                  " 5       U l        [        R                  " US[)        U5      -   -  USS9U l        U
" XbS9U l        [/        5       U l        g )Nr7   )get_normalizationFrM   )num_features)rO   rP   rQ   r_  r,   rZ  r\  r.   r?  r   rc   rd   re   rf   rj   to_qkv_multiscalerl   rH  ReLUnonlinearityr   rk   norm_outSanaMultiscaleAttnProcessor2_0r0   )ru   rI  rY  rJ  rZ  r[  r\  r]  r,   r.   r_  rR   rK  rw   s                rx   rP   &SanaMultiscaleLinearAttention.__init__c  s    	 	5"4"#6  >Q=XC1D89^q 	 (<	IIk5A	IIk5A	IIk5A	!#'K""))1)R]^ (
 GGIii	Q\1B-B C\X]^))O79rz   r   r  r  r}   c                 :   [         R                  " USSSS9n[        R                  " X2R	                  SS5      5      n[        R                  " XA5      nUR                  [        R                  S9nUS S 2S S 2S S24   US S 2S S 2SS 24   U R                  -   -  nU$ )Nr   r   r   r7   constantr7   moder  rH   r  r   )rr   r  r   matmulr  r   float32r,   ru   r   r  r  scoresr   s         rx   apply_linear_attention4SanaMultiscaleLinearAttention.apply_linear_attention  s    e\
!De]]2r%:;V3%((u}}(=%aCRCi0M!Q)4Ltxx4WXrz   c                 8   [         R                  " UR                  SS5      U5      nUR                  [         R                  S9nU[         R
                  " USSS9U R                  -   -  n[         R                  " X4R                  UR                  5      5      nU$ )NrH   r  rl  r	   T)rJ   keepdim)r   rm  r  r   rn  sumr,   r   ro  s         rx   apply_quadratic_attention7SanaMultiscaleLinearAttention.apply_quadratic_attention  sr    cmmB3U;/599VDADHHLMUIIekk,BCrz   r   c                 $    U R                  X5      $ r9  r   rU  s     rx   r   %SanaMultiscaleLinearAttention.forward  s    ~~d22rz   )rZ  r,   rc  rd  r\  r0   r.   re   rk   rd   ra  rf   )Nr6  r;   
batch_norm)   V瞯<F)r   r;  r<  r=  r>  r?  r   r  rA  r   r@  rP   r   rB  rq  rv  r   rD  rE  rF  s   @rx   rX  rX  `  s   3 .2"#%(,$)):): ): &c]	):
  ): ): ): CHo): ): "): ):VELL u|| TYT`T` ejeqeq u|| %,, W\WcWc hmhtht 3U\\ 3ell 3 3rz   rX  c                      ^  \ rS rSr          SS\S\SSS\S\S\S	\S
\S\\   S\\   S\S\S\4U 4S jjjr  SS\	R                  S\\	R                     S\\	R                     4S jjrSrU =r$ )MochiAttentioni  r   r%   r0   MochiAttnProcessor2_0r   r   r   r   r&   r2   r3   r)   rW   r,   c                   > [         TU ]  5         SSKJn  U	b  U	OXT-  U l        U	b  U	OUU l        U
(       a  U
OUU l        Xl        U	b  X-  OUU l        U" X]S5      U l	        U" X]S5      U l
        U" X]S5      U l        U" X]S5      U l        [        R                  " XR                  US9U l        [        R                  " XR                  US9U l        [        R                  " XR                  US9U l        [        R                  " X R                  US9U l        [        R                  " X R                  US9U l        U R                  b$  [        R                  " X R                  US9U l        [        R*                  " / 5      U l        U R,                  R/                  [        R                  " U R                  U R
                  US95        U R,                  R/                  [        R0                  " U5      5        U R                  (       d/  [        R                  " U R                  U R                  US9U l        X0l        g )Nr7   )MochiRMSNormTrM   )rO   rP   rQ   r  rR   r2   r3   rW   r   r_   r`   ro   rp   r   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   r0   )ru   r   r%   r0   r   r   r   r   r&   r2   r3   r)   rW   r,   r  rw   s                  rx   rP   MochiAttention.__init__  s     	/$+$7X=M")"5w92Ay 0,3,?W(U
"8$7"8$7(=(=IIidC	IIidC	IIidC	))$5~~O\))$5~~O\  , ii(9>>P_`DOmmB'299T^^T\\QR2::g./$$ ii8L8LS[\DO"rz   r   r   r   c                 0    U R                   " U U4UUS.UD6$ )Nr   r   )ru   r   r   r   kwargss        rx   r   MochiAttention.forward  s2     ~~
 #8)	

 
 	
rz   )rg   ri   rh   rW   r   rR   rp   ro   r`   r_   r3   r2   r0   rn   re   rk   rd   rf   )
r6  r7  r  FTNNTFrL   r:  )r   r;  r<  r=  r?  r  r@  r   rP   r   rB  r   rD  rE  rF  s   @rx   r~  r~    s      $!%)-!&/#/# /# +	/#
 /# /# /# /# /# #/# "#/# /# /# /# /#h 9=15	
||
  (5
 !.	
 
rz   r~  c                       \ rS rSrSrS r SSSS\R                  S\R                  S	\R                  S
\\R                     S\R                  4S jjr	Sr
g)r  i  z"Attention processor used in Mochi.c                 D    [        [        S5      (       d  [        S5      eg )NrN   zUMochiAttnProcessor2_0 requires PyTorch 2.0. To use it, please upgrade PyTorch to 2.0.rq   rr   ImportErrorru   s    rx   rP   MochiAttnProcessor2_0.__init__  !    q899uvv :rz   Nattnr~  r   r   r   image_rotary_embr}   c           	      :   UR                  U5      nUR                  U5      nUR                  U5      nUR                  SUR                  S45      nUR                  SUR                  S45      nUR                  SUR                  S45      nUR
                  b  UR                  U5      nUR                  b  UR                  U5      nUR                  U5      n	UR                  U5      n
UR                  U5      nU	R                  SUR                  S45      n	U
R                  SUR                  S45      n
UR                  SUR                  S45      nUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
Ub  S nU" U/UQ76 nU" U/UQ76 nUR                  SS5      UR                  SS5      UR                  SS5      pnU	R                  SS5      U
R                  SS5      UR                  SS5      pn	UR                  S5      nU	R                  S5      nX-   nUR                  u  nnnn/ n[        U5       GH&  nUU   S S S 24   n[         R"                  " UR%                  5       SS9R%                  5       nU	UUS-   2S S 2US S 24   nU
UUS-   2S S 2US S 24   nUUUS-   2S S 2US S 24   n[         R&                  " UUUS-    U/SS9n[         R&                  " UUUS-    U/SS9n[         R&                  " UUUS-    U/SS9n[(        R*                  " UUUSSS	9nUR                  S5      n[(        R,                  " US
S
S
UU-
  45      nUR/                  U5        GM)     [         R&                  " US
S9nUR                  SS5      R%                  SS5      nUR1                  X4SS9u  p#UR2                  S
   " U5      nUR2                  S   " U5      n[5        US5      (       a  UR7                  U5      nX#4$ )Nr	   rH   c                 4   U SSS S24   R                  5       nU SSS S24   R                  5       nX1-  XB-  -
  R                  U R                  5      nX2-  XA-  -   R                  U R                  5      n[        R                  " XV/SS9R                  S5      $ )N.r   r	   r7   rH   r  r  )r  r   r   r   stackflatten)x	freqs_cos	freqs_sinx_evenx_oddcossins          rx   apply_rotary_emb8MochiAttnProcessor2_0.__call__.<locals>.apply_rotary_emb  s    319++-#qt!t)**,)E,==AA!''J)E,==AA!''J{{C:26>>rBBrz   r7   F)as_tupler  r  	dropout_pr5   r   r   rn   )rd   re   rf   	unflattenr   r_   r`   ri   rg   rh   ro   rp   r  sizer   ranger   nonzeror  r  rr   rN   r  rl   split_with_sizesrk   rq   rn   ) ru   r  r   r   r   r  r   r  r  encoder_queryencoder_keyencoder_valuer  sequence_lengthencoder_sequence_lengthtotal_lengthr   r   r   rJ   attn_outputsidxmaskvalid_prompt_token_indicesvalid_encoder_queryvalid_encoder_keyvalid_encoder_valuevalid_query	valid_keyvalid_valueattn_outputvalid_sequence_lengths                                    rx   r   MochiAttnProcessor2_0.__call__  s,    		-(ii&		-(DJJ#34mmA

B/0DJJ#34;;"KK&E;;"++c"C(=>oo&;<(=>%//DJJ3CD!++A

B/?@%//DJJ3CD( --m<M(++K8K'C %U>-=>E"3:)9:C!OOAq13==A3FXY[\H]E##Aq)!!!Q'##Aq) %2  **Q-"/"4"4Q"7&@$)KK!
E1c$C!#&tQw/D).t||~PU)V)^)^)`&"/cAgqB\^_0_"` +C#'M1>XZ[,[ \"/cAgqB\^_0_"`))U3q%9;N$OUVWK		3sS1W#57H"IqQI))U3q%9;N$OUVWK88YseK %0$4$4Q$7!%%aA|F[7[-\]K,# %& 		,A6%//15==aC/</M/M6A 0N 0
,
 A}5A}54&&$(OO4I$J!33rz    r9  r   r;  r<  r=  r>  rP   r   rB  r   r   rD  r  rz   rx   r  r    su    ,w 48_4_4 ||_4  %||	_4
 _4 #5<<0_4 
_4 _4rz   r  c                       \ rS rSrSr   SS\S\R                  S\\R                     S\\R                     S\\R                     S	\R                  4S
 jjr	Sr
g)r1   iM  zB
Default processor for performing attention-related computations.
Nr  r   r   r   tembr}   c                    [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUR                  XOU5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR!                  UUU5      n["        R$                  " UU5      nUR'                  U5      nUR(                  S   " U5      nUR(                  S   " U5      nU
S:X  a$  UR                  SS	5      R+                  UWWW5      nUR,                  (       a  X)-   nX!R.                  -  nU$ )
Nr   rY   The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`.1.0.0r  r7   r	   rH   r  )r   getr   r^   r   r   viewr  r#  rK   rd   rb   r&  re   rf   r   r  r   bmmr   rk   r   r.   r-   )ru   r  r   r   r   r  argsr  deprecation_messageresidual
input_ndimr   channelheightwidthr  r   r   r  r  r  s                        rx   r   AttnProcessor.__call__R  s3    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q 44^V`a??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0&&u-$$S)&&u-33E3O		/59..}= A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r  NNNr   r;  r<  r=  r>  r   r   rB  r   r   rD  r  rz   rx   r1   r1   M  s     9=15'+@@ ||@  (5	@
 !.@ u||$@ 
@ @rz   r1   c                      ^  \ rS rSrSr      SS\S\S\\   S\\   S\S\4U 4S	 jjjr	  SS
\
S\R                  S\\R                     S\\R                     S\R                  4
S jjrSrU =r$ )r   i  a  
Processor for implementing attention for the Custom Diffusion method.

Args:
    train_kv (`bool`, defaults to `True`):
        Whether to newly train the key and value matrices corresponding to the text features.
    train_q_out (`bool`, defaults to `True`):
        Whether to newly train query matrices corresponding to the latent image features.
    hidden_size (`int`, *optional*, defaults to `None`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`, *optional*, defaults to `None`):
        The number of channels in the `encoder_hidden_states`.
    out_bias (`bool`, defaults to `True`):
        Whether to include the bias parameter in `train_q_out`.
    dropout (`float`, *optional*, defaults to 0.0):
        The dropout probability to use.
r   r   r   r   r)   r   c                 L  > [         TU ]  5         Xl        X l        X0l        X@l        U R                  (       aH  [        R                  " U=(       d    UUSS9U l        [        R                  " U=(       d    UUSS9U l	        U R                  (       a  [        R                  " X3SS9U l
        [        R                  " / 5      U l        U R                  R                  [        R                  " X3US95        U R                  R                  [        R                  " U5      5        g g NFrM   rO   rP   r   r   r   r   r   rc   r   to_v_custom_diffusionto_q_custom_diffusionrj   to_out_custom_diffusionrl   rm   ru   r   r   r   r   r)   r   rw   s          rx   rP   %CustomDiffusionAttnProcessor.__init__       	 &&#6  ==)+3F3U+Wbin)oD&)+3F3U+Wbin)oD&)+;RW)XD&+-==+<D(((//		+Ya0bc((//

70CD	 rz   r  r   r   r   r}   c                 ^   UR                   u  pVnUR                  XFU5      nU R                  (       a?  U R                  U5      R	                  UR
                  R                  R                  5      nO>UR                  UR	                  UR
                  R                  R                  5      5      nUc  Sn	UnO$Sn	UR                  (       a  UR                  U5      nU R                  (       a  U R                  UR	                  U R                  R                  R                  5      5      n
U R                  UR	                  U R                  R                  R                  5      5      nU
R	                  UR
                  R                  R                  5      n
UR	                  UR
                  R                  R                  5      nO"UR                  U5      n
UR                  U5      nU	(       ai  [        R                   " U
5      nUS S 2S S2S S 24   S-  US S 2S S2S S 24'   X-  SU-
  U
R#                  5       -  -   n
X-  SU-
  UR#                  5       -  -   nUR%                  U5      nUR%                  U
5      n
UR%                  U5      nUR'                  XU5      n[        R(                  " X5      nUR+                  U5      nU R                  (       a,  U R,                  S   " U5      nU R,                  S   " U5      nU$ UR.                  S   " U5      nUR.                  S   " U5      nU$ )NFTr7   r  r   )r   r#  r   r  r   rd   r   r   rb   r&  r   r   r  re   rf   r   	ones_likedetachr   r  r  r   r  rk   )ru   r  r   r   r   r   r  r   r   	crossattnr  r  r  r  s                 rx   r   %CustomDiffusionAttnProcessor.__call__  s    *7)<)<&
Q44^V`a..}=@@AQAQAWAWXEIIm..tyy/?/?/E/EFGE (I$1!I(,(G(GH](^%==,,-B-E-EdF`F`FgFgFmFm-noC../D/G/GHbHbHiHiHoHo/pqE&&))//0CHHTYY--334E))12CII34E__S)F%a!Qh/#5F1bqb!8,!f*

!<<CNa&jELLN%BBE&&u-$$S)&&u-33EO		/9..}= 88;MJM 88;MJM 	 !KKN=9M KKN=9Mrz   r   r   r   r  r  r  r   r   TTNNTr  r:  r   r;  r<  r=  r>  r@  r   r?  r  rP   r   r   rB  r   rD  rE  rF  s   @rx   r   r     s    (  %)-1EE E c]	E
 &c]E E E E< 9=1588 ||8  (5	8
 !.8 
8 8rz   r   c                       \ rS rSrSr  SS\S\R                  S\\R                     S\\R                     S\R                  4
S	 jjr	S
r
g)r   i  z{
Processor for performing attention-related computations with extra learnable key and value matrices for the text
encoder.
Nr  r   r   r   r}   c                    [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        UnUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pnUR                  XJU	5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  (       dr  UR                  U5      nUR!                  U5      nUR                  U5      nUR                  U5      n["        R$                  " X/SS9n["        R$                  " UU/SS9nOUnUnUR'                  XU5      n["        R(                  " UU5      nUR+                  U5      nUR,                  S   " U5      nUR,                  S   " U5      nUR                  SS	5      R/                  UR                  5      nX(-   nU$ )
Nr   rY   r  r  r7   rH   r	   r  r  )r   r  r   r  r   r  r#  rb   r&  rK   rd   r   rg   rh   r+   re   rf   r   r  r  r  r   rk   r   )ru   r  r   r   r   r  r  r  r  r   r  r   r   encoder_hidden_states_key_proj encoder_hidden_states_value_projr  r  r  s                     rx   r   AttnAddedKVProcessor.__call__  sI    t9q=FJJw5A #Ugw(;< %**=+>+>q+A=CVCVWXCY[]^hhijlmn)6)<)<&
Q44^V`a ($1!__$($C$CDY$Z!(?(?1(EFPPQRTUV		-(&&u-)-9N)O&+/??;P+Q()-)?)?@^)_&+/+A+ABb+c((())M*CIIm,E((-C**51E));AqICII?GQOE0C4E33EO		/59..}= A}5A}5%//B7??O%0rz   r  r:  r  r  rz   rx   r   r     sc     9=15:: ||:  (5	:
 !.: 
: :rz   r   c                       \ rS rSrSrS r  SS\S\R                  S\	\R                     S\	\R                     S	\R                  4
S
 jjr
Srg)r   i@  z
Processor for performing scaled dot-product attention (enabled by default if you're using PyTorch 2.0), with extra
learnable key and value matrices for the text encoder.
c                 D    [        [        S5      (       d  [        S5      eg )NrN   zWAttnAddedKVProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP    AttnAddedKVProcessor2_0.__init__F  %    q899i  :rz   Nr  r   r   r   r}   c           	         [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        UnUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pnUR                  XJU	SS	9nUc  UnO"UR                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR                  USS	9nUR                  U5      nUR                  U5      nUR                  USS	9nUR                  USS	9nUR                  (       dp  UR                  U5      nUR!                  U5      nUR                  USS	9nUR                  USS	9n["        R$                  " X/SS
9n["        R$                  " UU/SS
9nOUnUn[&        R(                  " XUUSSS9nUR                  SS5      R+                  U	SUR                  S   5      nUR,                  S   " U5      nUR,                  S   " U5      nUR                  SS5      R+                  UR                  5      nX(-   nU$ )Nr   rY   r  r  r7   rH   r	   r  )r2   r  r  F	attn_maskr  r5   r  )r   r  r   r  r   r  r#  rb   r&  rK   rd   r   rg   rh   r+   re   rf   r   r  rr   rN   r   rk   )ru   r  r   r   r   r  r  r  r  r   r  r   r   r  r  r  r  s                    rx   r    AttnAddedKVProcessor2_0.__call__L  s    t9q=FJJw5A #Ugw(;< %**=+>+>q+A=CVCVWXCY[]^hhijlmn)6)<)<&
Q44^V`jk4l ($1!__$($C$CDY$Z!(?(?1(EFPPQRTUV		-(&&ua&8)-9N)O&+/??;P+Q()-)?)?@^hi)?)j&+/+A+ABblm+A+n((())M*CIIm,E((a(8C**5!*<E));AqICII?GQOE0C4E 663RW
 &//15==j"hnn]^N_` A}5A}5%//B7??O%0rz   r  r:  r   r;  r<  r=  r>  rP   r   r   rB  r   r   rD  r  rz   rx   r   r   @  sh    
 9=15== ||=  (5	=
 !.= 
= =rz   r   c                       \ rS rSrSrS r  SS\S\R                  S\R                  S\	\R                     S	\R                  4
S
 jjr
Srg)r   i  YAttention processor used typically in processing the SD3-like self-attention projections.c                 D    [        [        S5      (       d  [        S5      eg )NrN   zUJointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   JointAttnProcessor2_0.__init__  r  rz   Nr  r   r   r   r}   c                    UnUR                   S   nUR                  U5      n	UR                  U5      n
UR                  U5      nU
R                   S   nXR                  -  nU	R                  USUR                  U5      R                  SS5      n	U
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
UGb<  UR                  U5      nUR                  U5      nUR                  U5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " X/SS9n	[        R                  " X/SS9n
[        R                  " UU/SS9n[         R"                  " XUSSS9nUR                  SS5      R%                  USUR                  U-  5      nUR'                  U	R(                  5      nUbQ  US S 2S UR                   S   24   US S 2UR                   S   S 24   p2UR*                  (       d  UR-                  U5      nUR.                  S   " U5      nUR.                  S   " U5      nUb  X#4$ U$ )	Nr   rH   r7   r	   r  r  Fr  )r   rd   re   rf   r   r  r  r_   r`   ri   rg   rh   ro   rp   r   r  rr   rN   r   r   r   rW   rn   rk   )ru   r  r   r   r   r  r  r  r   r   r  r  rR   head_dim encoder_hidden_states_query_projr  r  s                    rx   r   JointAttnProcessor2_0.__call__  s%    !"((+
 		-(ii&		-(IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C !,/3?T/U,-1__=R-S*/3?T/U,/O/T/TB

H0i1o - .L-P-PB

H.i1o + 0P/T/TB

H0i1o -   ,373D3DEe3f0  ,151B1BCa1b.IIuGQOE))SAqICIIu&FGQOE66u5TWchi%//15==j"djj[cNcd%((5 , a!48>>!#4!445a!2!445 1 (((,8M(N% A}5A}5 , 77  rz   r  r:  r   r;  r<  r=  r>  rP   r   r   FloatTensorr   r   rD  r  rz   rx   r   r     sq    cw 486:L!L! ((L!  %00	L!
 !!2!23L! 
		L! L!rz   r   c                       \ rS rSrSrS r  SS\S\R                  S\R                  S\	\R                     S	\R                  4
S
 jjr
Srg)PAGJointAttnProcessor2_0i  r  c                 D    [        [        S5      (       d  [        S5      eg )NrN   zXPAGJointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   !PAGJointAttnProcessor2_0.__init__  s%    q899j  :rz   Nr  r   r   r   r}   c           	      T   UnUR                   nUS:X  a3  UR                  u  pxpUR                  XxX-  5      R                  SS5      nUR                   nUS:X  a3  UR                  u  pxpUR                  XxX-  5      R                  SS5      nUR                  S   nUR	                  S5      u  pUR	                  S5      u  nnUR                  S   nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                  " UU/SS9n[        R                  " UU/SS9n[        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      n[        R                  " UUUSSS	9nUR                  SS5      R!                  USUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   pUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      nUR                  S   nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                  " UU/SS9n[        R                  " UU/SS9n[        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR-                  S5      n[        R.                  " UU4UR0                  UR$                  S9n [3        S5      U S U2S U24'   U S U2S U24   R5                  S5        U R7                  S5      R7                  S5      n [        R                  " UUUU SSS9nUR                  SS5      R!                  USUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   nnUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      nUS:X  a$  UR                  SS
5      R!                  UWW	W
5      n[        R                  " X/5      n[        R                  " UU/5      nX#4$ )Nr  r7   r	   r   r  rH   r  Fr  r  r   -infr  )r   r   r  r  chunkrd   re   rf   ri   rg   rh   r   r  r   rr   rN   r   r   r   rk   rW   rn   r  r  r   r  fill_diagonal_r  )!ru   r  r   r   r   r  r  r   r  r  r  context_input_ndimidentity_block_sizehidden_states_orghidden_states_ptbencoder_hidden_states_orgencoder_hidden_states_ptb	query_orgkey_org	value_org$encoder_hidden_states_org_query_proj"encoder_hidden_states_org_key_proj$encoder_hidden_states_org_value_projrR   r  	query_ptbkey_ptb	value_ptb$encoder_hidden_states_ptb_query_proj"encoder_hidden_states_ptb_key_proj$encoder_hidden_states_ptb_value_projr   	full_masks!                                    rx   r   !PAGJointAttnProcessor2_0.__call__  s    !"''
?1>1D1D.J)..zFNS]]^_abcM277"1F1L1L.J$9$>$>zTZTb$c$m$mnoqr$s! ,11!4 0=/B/B1/E,?T?Z?Z[\?]<!#< /44Q7
 II/0	))-.II/0	 04?X/Y,-1__=V-W*/3?X/Y, IIy*NOUVW	))W&HIqQIIy*NOUVW	MM"%	

*NN:r4::xHRRSTVWX	,,z2tzz8DNNqRSTNN:r4::xHRRSTVWX	::w	SE
 .771=EEjRTVZV`V`ckVkl-00A a!48>>!#4!445a!2!445 5 !KKN+<= KKN+<=$$(,8Q(R%? 1 ; ;B C K KJX_agin o"(A(K(KBPR(S([([GVU)% /44Q7
 II/0	))-.II/0	 04?X/Y,-1__=V-W*/3?X/Y, IIy*NOUVW	))W&HIqQIIy*NOUVW	MM"%	

*NN:r4::xHRRSTVWX	,,z2tzz8DNNqRSTNN:r4::xHRRSTVWX	 ..#KK' 29;K;KS\SbSbc	 AFf	&&&(<)<(<<= 	&&&(<)<(<<=LLQO ''*44Q7	::w	Y#Y^
 .771=EEjRTVZV`V`ckVkl-00A a!48>>!#4!445a!2!445 5 !KKN+<= KKN+<=$$(,8Q(R%? 1 ; ;B C K KJX_agin o"(A(K(KBPR(S([([GVU)%
 		#4"HI %		+DF_*` a33rz   r  r:  r  r  rz   rx   r  r    sp    c 486:P4P4 ((P4  %00	P4
 !!2!23P4 
		P4 P4rz   r  c                       \ rS rSrSrS r  SS\S\R                  S\R                  S\	\R                     S	\R                  4
S
 jjr
Srg)PAGCFGJointAttnProcessor2_0i~  r  c                 D    [        [        S5      (       d  [        S5      eg )NrN   z[PAGCFGJointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   $PAGCFGJointAttnProcessor2_0.__init__  %    q899m  :rz   Nr  r   r   r   r}   c           	         UnUR                   nUS:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                   nUS:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                  S   nUR	                  S5      u  nnn[
        R                  " UU/5      nUR	                  S5      u  nnn[
        R                  " UU/5      nUR                  S   n	UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[
        R                  " UU/SS9n[
        R                  " UU/SS9n[
        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      n[        R                  " UUUSS	S
9nUR                  SS5      R!                  U	SUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   nnUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      nUR                  S   n	UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n UR                  U5      n!UR                  U5      n"[
        R                  " UU /SS9n[
        R                  " UU!/SS9n[
        R                  " UU"/SS9nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR-                  S5      n#[
        R.                  " U#U#4UR0                  UR$                  S9n$[3        S5      U$S U2S U24'   U$S U2S U24   R5                  S5        U$R7                  S5      R7                  S5      n$[        R                  " UUUU$SS	S9nUR                  SS5      R!                  U	SUR                  U-  5      nUR#                  UR$                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   nnUR&                  S   " U5      nUR&                  S   " U5      nUR(                  (       d  UR+                  U5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      nUS:X  a$  UR                  SS5      R!                  U	W
WW5      n[
        R                  " UU/5      n[
        R                  " UU/5      nX#4$ )Nr  r7   r	   r   r   r  rH   r  Fr  r  r   r  r  )r   r   r  r  r  r   r  rd   re   rf   ri   rg   rh   r   rr   rN   r   r   r   rk   rW   rn   r  r  r   r  r  r  )%ru   r  r   r   r   r  r  r  r  r   r  r  r  r  r  hidden_states_uncondr  r   encoder_hidden_states_uncondr  r  r  r  r  r  r  r  rR   r  r	  r
  r  r  r  r  r   r  s%                                        rx   r   $PAGCFGJointAttnProcessor2_0.__call__  s    !"''
?1>1D1D.J)..zFNS]]^_abcM277"1F1L1L.J$9$>$>zTZTb$c$m$mnoqr$s!+11

 FSEXEXYZE[B/1B!II';=N&OP "''*		
(%%$)II/KMf.g$h! /44Q7
 II/0	))-.II/0	 04?X/Y,-1__=V-W*/3?X/Y, IIy*NOUVW	))W&HIqQIIy*NOUVW	MM"%	

*NN:r4::xHRRSTVWX	,,z2tzz8DNNqRSTNN:r4::xHRRSTVWX	::w	SE
 .771=EEjRTVZV`V`ckVkl-00A a!48>>!#4!445a!2!445 5 !KKN+<= KKN+<=$$(,8Q(R%? 1 ; ;B C K KJX_agin o"(A(K(KBPR(S([([GVU)% /44Q7
 II/0	))-.II/0	 04?X/Y,-1__=V-W*/3?X/Y, IIy*NOUVW	))W&HIqQIIy*NOUVW	MM"%	

*NN:r4::xHRRSTVWX	,,z2tzz8DNNqRSTNN:r4::xHRRSTVWX	 ..#KK' 29;K;KS\SbSbc	 AFf	&&&(<)<(<<= 	&&&(<)<(<<=LLQO ''*44Q7	::w	Y#Y^
 .771=EEjRTVZV`V`ckVkl-00A a!48>>!#4!445a!2!445 5 !KKN+<= KKN+<=$$(,8Q(R%? 1 ; ;B C K KJX_agin o"(A(K(KBPR(S([([GVU)%
 		#46G"HI %		+DF_*` a33rz   r  r:  r  r  rz   rx   r  r  ~  sp    c 486:Y4Y4 ((Y4  %00	Y4
 !!2!23Y4 
		Y4 Y4rz   r  c                       \ rS rSrSrS r  SS\S\R                  S\R                  S\	\R                     S	\R                  4
S
 jjr
Srg)FusedJointAttnProcessor2_0i#  r  c                 D    [        [        S5      (       d  [        S5      eg NrN   zPAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   #FusedJointAttnProcessor2_0.__init__&  !    q899pqq :rz   Nr  r   r   r   r}   c                 n   UnUR                   nUS:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                   nUS:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                  S   n	UR	                  U5      nUR                  S   S-  n[
        R                  " XSS9u  nnnUR                  U5      nUR                  S   S-  n[
        R                  " UUSS9u  nnn[
        R                  " UU/SS9n[
        R                  " UU/SS9n[
        R                  " UU/SS9nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      nUR                  U	SUR                  U5      R                  SS5      n[        R                  " UUUSS	S
9nUR                  SS5      R                  U	SUR                  U-  5      nUR                  UR                  5      nUS S 2S UR                  S   24   US S 2UR                  S   S 24   p2UR                  S   " U5      nUR                  S   " U5      nUR                   (       d  UR#                  U5      nUS:X  a$  UR                  SS5      R                  U	W
WW5      nUS:X  a$  UR                  SS5      R                  U	W
WW5      nX#4$ )Nr  r7   r	   r   rH   r   r  r  Fr  r  )r   r   r  r  r*  r   splitr.  r  r   rr   rN   r   r   r   rk   rW   rn   )ru   r  r   r   r   r  r  r  r  r   r  r  r  r  qkv
split_sizer   r  r  encoder_qkvr  r  r  rR   r  s                            rx   r   #FusedJointAttnProcessor2_0.__call__*  s1    !"''
?1>1D1D.J)..zFNS]]^_abcM277"1F1L1L.J$9$>$>zTZTb$c$m$mnoqr$s!*003
 kk-(YYr]a'
!KKR@sE ''(=> &&r*a/

 KKZR8		
,*, 		5"BCKii<=1E		5"BCKIIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP66uc5TWchi%//15==j"djj[cNcd%((5 !0x~~a0001!X^^A.001 - A}5A}5$$$(OO4I$J!?)33B;CCJPWY_afgM"$9$C$CB$K$S$ST^`gioqv$w!33rz   r  r:  r  r  rz   rx   r  r  #  sq    cr 486:E4E4 ((E4  %00	E4
 !!2!23E4 
		E4 E4rz   r  c                       \ rS rSrSrSS\\   4S jjr  SS\S\	R                  S\	R                  S	\\	R                     S
\	R                  4
S jjrSrg)r   ir    
Processor for implementing memory efficient attention using xFormers.

Args:
    attention_op (`Callable`, *optional*, defaults to `None`):
        The base
        [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
        use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
        operator.
Nr   c                     Xl         g r9  r   ru   r   s     rx   rP   #XFormersJointAttnProcessor.__init__~      (rz   r  r   r   r   r}   c           	         UnUR                  U5      nUR                  U5      n	UR                  U5      n
UR                  U5      R	                  5       nUR                  U	5      R	                  5       n	UR                  U
5      R	                  5       n
UR
                  b  UR                  U5      nUR                  b  UR                  U	5      n	UGb  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      R	                  5       nUR                  U5      R	                  5       nUR                  U5      R	                  5       nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " X/SS9n[        R                  " X/SS9n	[        R                  " X/SS9n
[        R                  R!                  XXU R"                  UR$                  S9nUR'                  UR(                  5      nUR+                  U5      nUbQ  US S 2S UR,                  S   24   US S 2UR,                  S   S 24   p2UR.                  (       d  UR1                  U5      nUR2                  S   " U5      nUR2                  S   " U5      nUb  X#4$ U$ )Nr7   r  	attn_biasoprY   r   )rd   re   rf   r   
contiguousr_   r`   ri   rg   rh   ro   rp   r   r  r   r   r   r   rY   r   r   r   r   rW   rn   rk   )ru   r  r   r   r   r  r  r  r   r  r  r  r  r  s                 rx   r   #XFormersJointAttnProcessor.__call__  s    ! 		-(ii&		-(&&u-88:$$S)446&&u-88:;;"KK&E;;"++c"C !,/3?T/U,-1__=R-S*/3?T/U,/3/E/EFf/g/r/r/t,-1-C-CDb-c-n-n-p*/3/E/EFf/g/r/r/t,  ,373D3DEe3f0  ,151B1BCa1b.IIuGQOE))SAqICIIuGQOE ??D<M<MUYU_U_ @ 
 &((5..}= , a!48>>!#4!445a!2!445 1 (((,8M(N% A}5A}5 , 77  rz   r   r9  r:  )r   r;  r<  r=  r>  r   r   rP   r   r   r  r   rD  r  rz   rx   r   r   r  s~    	)Xh%7 ) 486:C!C! ((C!  %00	C!
 !!2!23C! 
		C! C!rz   r   c                       \ rS rSrSrS r    SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\	\R                     S\R                  4S jjr
Srg)AllegroAttnProcessor2_0i  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the Allegro model. It applies a normalization layer and rotary embedding on the query and key vector.
c                 D    [        [        S5      (       d  [        S5      eg )NrN   zWAllegroAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP    AllegroAttnProcessor2_0.__init__  r  rz   Nr  r   r   r   r  r  r}   c           	      4   UnUR                   b  UR                  X%5      nUR                  nUS:X  a3  UR                  u  ppUR                  XX-  5      R	                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XMU	5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U	SUR                  U5      R	                  SS5      nUR                  U	SUR                  U5      R	                  SS5      nUR                  U	SUR                  U5      R	                  SS5      nUb6  UR                  (       d%  SSKJn  U" XS   US   5      nU" UUS   US   5      n[         R"                  " UUUUSSS	9nUR	                  SS5      R%                  U	SUR                  U-  5      nUR'                  UR(                  5      nUR*                  S   " U5      nUR*                  S   " U5      nUS:X  a$  UR	                  SS
5      R%                  U	W
WW5      nUR,                  (       a  X'-   nX!R.                  -  nU$ )Nr  r7   r	   rH   )apply_rotary_emb_allegror   r  Fr  r  )r^   r   r   r  r  r#  r   rK   rd   rb   r&  re   rf   rU   
embeddingsr7  rr   rN   r   r   r   rk   r.   r-   )ru   r  r   r   r   r  r  r  r  r   r  r  r  r  r   r   r  r  rR   r  r7  s                        rx   r    AllegroAttnProcessor2_0.__call__  s    !( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP '0G0G<,UQ4GIYZ[I\]E*30@0CEUVWEXYC 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r  NNNNr  r  rz   rx   r3  r3    s    
 9=15'+37OO ||O  (5	O
 !.O u||$O #5<<0O 
O Orz   r3  c            	       z    \ rS rSrSrS r SS\S\R                  S\R                  S\R                  4S	 jjr	S
r
g)AuraFlowAttnProcessor2_0i%  z;Attention processor used typically in processing Aura Flow.c                 h    [        [        S5      (       d  [        SS5      (       a  [        S5      eg g )NrN   r   2.1zAuraFlowAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to at least 2.1 or above as we use `scale` in `F.scaled_dot_product_attention()`. rq   rr   r   r  r  s    rx   rP   !AuraFlowAttnProcessor2_0.__init__(  s:    q899>NsTY>Z>Z v  ?[9rz   Nr  r   r   r}   c           	      D   UR                   S   nUR                  U5      nUR                  U5      nUR                  U5      n	Ub3  UR	                  U5      n
UR                  U5      nUR                  U5      nUR                   S   nXR                  -  nUR                  USUR                  U5      nUR                  USUR                  U5      nU	R                  USUR                  U5      n	UR                  b  UR                  U5      nUR                  b  UR                  U5      nUb  W
R                  USUR                  U5      n
WR                  USUR                  U5      nWR                  USUR                  U5      nUR                  b  UR                  U
5      n
UR                  b  UR                  U5      n[        R                  " X/SS9n[        R                  " X/SS9n[        R                  " X/SS9n	UR                  SS5      nUR                  SS5      nU	R                  SS5      n	[         R"                  " XxU	SUR$                  SS9nUR                  SS5      R'                  USUR                  U-  5      nUR)                  UR*                  5      nUb/  US S 2UR                   S   S 24   US S 2S UR                   S   24   p2UR,                  S   " U5      nUR,                  S   " U5      nUb  UR/                  U5      nUb  X#4$ U$ )	Nr   rH   r7   r  r	   r  Fr  rY   r5   )r   rd   re   rf   ri   rg   rh   r   r  r_   r`   ro   rp   r   r  r  rr   rN   rY   r   r   r   rk   rn   )ru   r  r   r   r  r  r   r   r  r  r  r  r  rR   r  s                  rx   r   !AuraFlowAttnProcessor2_0.__call__.  s    #((+
 		-(ii&		-( !,/3?T/U,-1__=R-S*/3?T/U, IIbM	

*

:r4::x@hhz2tzz8<

:r4::x@ ;;"KK&E;;"++c"C !,/O/T/TB

H0, .L-P-PQ[]_aeakakmu-v*/O/T/TB

H0,   ,373D3DEe3f0  ,151B1BCa1b.II?GQOE));AqICII?GQOE1%mmAq!1% 66DJJ%
 &//15==j"djj[cNcd%((5 !,a!6!<!<Q!?!AABa!A#8#>#>q#A!AAB 1 A}5A}5 ,$(OO4I$J! , 77  rz   r  r9  r   r;  r<  r=  r>  rP   r   r   r  r   rD  r  rz   rx   r<  r<  %  sW    E 48	Q!Q! ((Q!  %00	Q! 
		Q! Q!rz   r<  c            	       z    \ rS rSrSrS r SS\S\R                  S\R                  S\R                  4S	 jjr	S
r
g)FusedAuraFlowAttnProcessor2_0i  zRAttention processor used typically in processing Aura Flow with fused projections.c                 h    [        [        S5      (       d  [        SS5      (       a  [        S5      eg g )NrN   r   r>  zFusedAuraFlowAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to at least 2.1 or above as we use `scale` in `F.scaled_dot_product_attention()`. r?  r  s    rx   rP   &FusedAuraFlowAttnProcessor2_0.__init__  s:    q899>NsTY>Z>Z {  ?[9rz   Nr  r   r   r}   c           	      h   UR                   S   nUR                  U5      nUR                   S   S-  n[        R                  " XxSS9u  pnUb<  UR	                  U5      nUR                   S   S-  n[        R                  " XSS9u  nnnU
R                   S   nUUR
                  -  nU	R                  USUR
                  U5      n	U
R                  USUR
                  U5      n
UR                  USUR
                  U5      nUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
Ub  WR                  USUR
                  U5      nWR                  USUR
                  U5      nWR                  USUR
                  U5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " X/SS9n	[        R                  " X/SS9n
[        R                  " X/SS9nU	R                  SS5      n	U
R                  SS5      n
UR                  SS5      n[        R                  " XUSUR                  SS	9nUR                  SS5      R!                  USUR
                  U-  5      nUR#                  U	R$                  5      nUb/  US S 2UR                   S   S 24   US S 2S UR                   S   24   p2UR&                  S   " U5      nUR&                  S   " U5      nUb  UR)                  U5      nUb  X#4$ U$ )
Nr   rH   r   r  r7   r	   r  FrB  )r   r*  r   r!  r.  r   r  r_   r`   ro   rp   r  r  rr   rN   rY   r   r   r   rk   rn   )ru   r  r   r   r  r  r   r"  r#  r   r  r  r$  r  r  r  rR   r  s                     rx   r   &FusedAuraFlowAttnProcessor2_0.__call__  s;    #((+
 kk-(YYr]a'
!KKR@E !,++,ABK$**2.!3J
 K<	0.0 IIbM	

*

:r4::x@hhz2tzz8<

:r4::x@ ;;"KK&E;;"++c"C !,/O/T/TB

H0, .L-P-PQ[]_aeakakmu-v*/O/T/TB

H0,   ,373D3DEe3f0  ,151B1BCa1b.II?GQOE));AqICII?GQOE1%mmAq!1% 66DJJ%
 &//15==j"djj[cNcd%((5 !,a!6!<!<Q!?!AABa!A#8#>#>q#A!AAB 1 A}5A}5 ,$(OO4I$J! , 77  rz   r  r9  rD  r  rz   rx   rF  rF    sW    \ 48	U!U! ((U!  %00	U! 
		U! U!rz   rF  c                       \ rS rSrSrS r  SS\S\R                  S\R                  S\	\R                     S	\	\R                     S
\R                  4S jjr
Srg)CogVideoXAttnProcessor2_0i  
Processor for implementing scaled dot-product attention for the CogVideoX model. It applies a rotary embedding on
query and key vectors, but does not include spatial normalization.
c                 D    [        [        S5      (       d  [        S5      eg NrN   zVCogVideoXAttnProcessor requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   "CogVideoXAttnProcessor2_0.__init__  !    q899vww :rz   Nr  r   r   r   r  r}   c           	         UR                  S5      n[        R                  " X2/SS9nUR                  u  pxn	Ub<  UR	                  XHU5      nUR                  XqR                  SUR                  S   5      nUR                  U5      n
UR                  U5      nUR                  U5      nUR                  S   nXR                  -  nU
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U
5      n
UR                  b  UR                  U5      nUbY  SSKJn  U" U
S S 2S S 2US 24   U5      U
S S 2S S 2US 24'   UR                  (       d!  U" US S 2S S 2US 24   U5      US S 2S S 2US 24'   [         R"                  " XXSSS9nUR                  SS5      R%                  USUR                  U-  5      nUR&                  S	   " U5      nUR&                  S   " U5      nUR)                  XbR                  S5      U-
  /SS9u  p2X#4$ )
Nr7   r  rH   r	   r  r  Fr  r   )r  r   r  r   r#  r  r   rd   re   rf   r  r_   r`   r8  r  rU   rr   rN   r   rk   r!  )ru   r  r   r   r   r  text_seq_lengthr   r  r   r   r  r  rR   r  r  s                   rx   r   "CogVideoXAttnProcessor2_0.__call__  sk    044Q7		#8"HaP)6)<)<&
Q%!88ZdeN+00ZZ^MaMabdMefN		-(ii&		-(IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '4,<U1aIYCY=Z\l,mE!Q(()**.>s1aIYCY?Z\l.mAq/**+663RW
 &//15==j"djj[cNcd A}5A}5/</B/B003oEFA 0C 0
, 33rz   r  r:  r  r  rz   rx   rL  rL    sw    
x 26378484 ||84  %||	84
 !.84 #5<<084 
84 84rz   rL  c                       \ rS rSrSrS r  SS\S\R                  S\R                  S\	\R                     S	\	\R                     S
\R                  4S jjr
Srg)FusedCogVideoXAttnProcessor2_0i(	  rM  c                 D    [        [        S5      (       d  [        S5      eg rO  r  r  s    rx   rP   'FusedCogVideoXAttnProcessor2_0.__init__.	  rQ  rz   Nr  r   r   r   r  r}   c           	      0   UR                  S5      n[        R                  " X2/SS9nUc  UR                  OUR                  u  pxn	Ub<  UR	                  XHU5      nUR                  XqR                  SUR                  S   5      nUR                  U5      n
U
R                  S   S-  n[        R                  " XSS9u  pnUR                  S   nXR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      nUbY  SSKJn  U" US S 2S S 2US 24   U5      US S 2S S 2US 24'   UR                  (       d!  U" US S 2S S 2US 24   U5      US S 2S S 2US 24'   [        R                   " XXSSS	9nUR                  SS5      R#                  USUR                  U-  5      nUR$                  S
   " U5      nUR$                  S   " U5      nUR                  XbR                  S5      U-
  /SS9u  p2X#4$ )Nr7   r  rH   r   r	   rS  r  Fr  r   )r  r   r  r   r#  r  r   r*  r!  r  r_   r`   r8  r  rU   rr   rN   r   rk   )ru   r  r   r   r   r  rT  r   r  r   r"  r#  r   r  r  rR   r  r  s                     rx   r   'FusedCogVideoXAttnProcessor2_0.__call__2	  s    044Q7		#8"HaP $9#@MF[FaFa 	'
Q %!88ZdeN+00ZZ^MaMabdMefNkk-(YYr]a'
!KKR@EIIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '4,<U1aIYCY=Z\l,mE!Q(()**.>s1aIYCY?Z\l.mAq/**+663RW
 &//15==j"djj[cNcd A}5A}5/</B/B003oEFA 0C 0
, 33rz   r  r:  r  r  rz   rx   rW  rW  (	  sw    
x 2637:4:4 ||:4  %||	:4
 !.:4 #5<<0:4 
:4 :4rz   rW  c                       \ rS rSrSrSS\\   4S jjr  SS\S\	R                  S\\	R                     S	\\	R                     S
\	R                  4
S jjrSrg)r   io	  r'  Nr   c                     Xl         g r9  r   r)  s     rx   rP   %XFormersAttnAddedKVProcessor.__init__{	  r+  rz   r  r   r   r   r}   c           	         UnUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pgnUR                  XGU5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      n	UR                  U	5      n	UR                  U5      n
UR                  U5      nUR                  U
5      n
UR                  U5      nUR                  (       dq  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                  " X/SS9n[        R                  " X/SS9nOU
nUn[         R"                  R%                  XXU R&                  UR(                  S9nUR+                  U	R,                  5      nUR/                  U5      nUR0                  S   " U5      nUR0                  S   " U5      nUR                  SS5      R3                  UR                  5      nX%-   nU$ )Nr   r7   rH   r	   r  r-  r  )r  r   r  r#  rb   r&  rK   rd   r   rg   rh   r+   re   rf   r   r  r   r   r   r   rY   r   r   r   rk   r   )ru   r  r   r   r   r  r   r  r   r   r  r  r  r  s                 rx   r   %XFormersAttnAddedKVProcessor.__call__~	  s:    !%**=+>+>q+A=CVCVWXCY[]^hhijlmn)6)<)<&
Q44^V`a ($1!__$($C$CDY$Z!(?(?1(EFPPQRTUV		-(&&u-)-9N)O&+/??;P+Q()-)?)?@^)_&+/+A+ABb+c((())M*CIIm,E((-C**51E));AqICII?GQOE0C4E ??D<M<MUYU_U_ @ 
 &((5..}= A}5A}5%//B7??O%0rz   r   r9  r:  r   r;  r<  r=  r>  r   r   rP   r   r   rB  r   rD  r  rz   rx   r   r   o	  st    	)Xh%7 ) 9=1555 ||5  (5	5
 !.5 
5 5rz   r   c                       \ rS rSrSrSS\\   4S jjr   SS\S\	R                  S\\	R                     S	\\	R                     S
\\	R                     S\	R                  4S jjrSrg)r   i	  r'  Nr   c                     Xl         g r9  r   r)  s     rx   rP   XFormersAttnProcessor.__init__	  r+  rz   r  r   r   r   r  r}   c           	      z   [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUR                  XOU5      nUb#  UR
                  u  nnnUR                  SUS5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR!                  U5      R#                  5       nUR!                  U5      R#                  5       nUR!                  U5      R#                  5       n[$        R&                  R)                  UUUX@R*                  UR,                  S	9nUR/                  UR0                  5      nUR3                  U5      nUR4                  S   " U5      nUR4                  S   " U5      nU
S:X  a$  UR                  SS
5      R7                  UWWW5      nUR8                  (       a  X)-   nX!R:                  -  nU$ )Nr   rY   r  r  r  r7   r	   rH   r-  r  )r   r  r   r^   r   r   r  r  r#  expandrK   rd   rb   r&  re   rf   r   r0  r   r   r   r   rY   r   r   r   rk   r   r.   r-   )ru   r  r   r   r   r  r  r  r  r  r  r   r  r  r  
key_tokensr   query_tokensr   r  r  s                        rx   r   XFormersAttnProcessor.__call__	  s    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	"
 44^Q[\% "/!4!4A|Q+222|RHN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0&&u-88:$$S)446&&u-88: ??3<M<MUYU_U_ @ 
 &((5..}= A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r   r9  r  ra  r  rz   rx   r   r   	  s    	)Xh%7 ) 9=15'+LL ||L  (5	L
 !.L u||$L 
L Lrz   r   c                       \ rS rSrSrS r   SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\R                  4S jjr
Srg)r   i
  z
Processor for implementing flash attention using torch_npu. Torch_npu supports only fp16 and bf16 data types. If
fp32 is used, F.scaled_dot_product_attention will be used for computation, but the acceleration effect on NPU is
not significant.

c                 8    [        5       (       d  [        S5      eg )NzTAttnProcessorNPU requires torch_npu extensions and is supported only on npu devices.)r   r  r  s    rx   rP   AttnProcessorNPU.__init__
  s    %''tuu (rz   Nr  r   r   r   r  r}   c                    [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUb  UR                  XOU5      nUR                  XR                  SUR
                  S   5      nUR                  SSUR
                  S   S5      nUR                  [        R                  :X  a%  [        R                  " UR                  5       5      nOUR                  5       nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR!                  U5      nUc  UnO"UR"                  (       a  UR%                  U5      nUR'                  U5      nUR)                  U5      nUR
                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  [        R*                  [        R,                  4;   aR  [.        R0                  " UUUUR                  S	S US
[2        R4                  " UR
                  S   5      -  SSS
SSS9S   nO[6        R8                  " UUUUSSS9nUR                  SS5      R;                  USUR                  U-  5      nUR=                  UR                  5      nUR>                  S   " U5      nUR>                  S   " U5      nU
S:X  a$  UR                  SS5      R;                  UWWW5      nUR@                  (       a  X)-   nX!RB                  -  nU$ )Nr   rY   r  r  r  r7   r	   rH   BNSDr;   i   F)	input_layoutpse
atten_maskrY   pre_tockensnext_tockens	keep_probsyncinner_preciser  r  r  )"r   r  r   r^   r   r   r  r  r#  r   repeatr   r   r@  logical_notrK   rd   rb   r&  re   rf   float16bfloat16	torch_npunpu_fusion_attentionmathsqrtrr   rN   r   r   rk   r.   r-   ru   r  r   r   r   r  r  r  r  r  r  r   r  r  r  r  r   r   r  r  rR   r  s                         rx   r   AttnProcessorNPU.__call__ 
  s    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN+221a9L9LQ9OQRSN##uzz1!&!2!2>3F3F3H!I!/!4!4!6??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP ;;5==%..99%::

#)DIIekk"o66!" M" ::sE^sV[M &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r  r  r  r  rz   rx   r   r   
  s    v 9=15'+cc ||c  (5	c
 !.c u||$c 
c crz   r   c                       \ rS rSrSrS r   SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\R                  4S jjr
Srg)rs   i
  k
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
c                 D    [        [        S5      (       d  [        S5      eg r  r  r  s    rx   rP   AttnProcessor2_0.__init__
  r  rz   Nr  r   r   r   r  r}   c           	         [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUb<  UR                  XOU5      nUR                  XR                  SUR
                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR
                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                   b  UR!                  U5      nUR"                  b  UR#                  U5      n[$        R&                  " UUUUS	S
S9nUR                  SS5      R)                  USUR                  U-  5      nUR+                  UR,                  5      nUR.                  S   " U5      nUR.                  S   " U5      nU
S:X  a$  UR                  SS5      R)                  UWWW5      nUR0                  (       a  X)-   nX!R2                  -  nU$ )Nr   rY   r  r  r  r7   r	   rH   r  Fr  r  )r   r  r   r^   r   r   r  r  r#  r   rK   rd   rb   r&  re   rf   r_   r`   rr   rN   r   r   r   rk   r.   r-   r  s                         rx   r   AttnProcessor2_0.__call__
  s    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r  r  r  r  rz   rx   rs   rs   
  s    r 9=15'+RR ||R  (5	R
 !.R u||$R 
R Rrz   rs   c                       \ rS rSrSrSS\\\\   S4      4S jjr   SS\	S\
R                  S	\\
R                     S
\\
R                     S\\
R                     S\
R                  4S jjrSrg)r   i
  r
Processor for implementing scaled dot-product attention with pallas flash attention kernel if using `torch_xla`.
Nr|   .c                     [        [        S5      (       d  [        S5      e[        SS5      (       a  [        S5      e[	        5       (       a  [        SS5      (       a  [        S5      eXl        g )NrN   zXXLAFlashAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r   r   6XLA flash attention requires torch_xla version >= 2.3.r   DSPMD support for XLA flash attention needs torch_xla version >= 2.4.)rq   rr   r  r   r   r|   )ru   r|   s     rx   rP   !XLAFlashAttnProcessor2_0.__init__
  sa    q899j   U++VWW99-c599dee,rz   r  r   r   r   r  r}   c           	         UnUR                   b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R	                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XNU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R	                  SS5      nUR                  U
SUR                  U5      R	                  SS5      nUR                  U
SUR                  U5      R	                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        S UUU4 5       5      (       a  Ubn  UR                  U
SSUR                  S   5      nUR!                  5       R#                  US:H  [!        S5      5      R#                  US:H  [!        S5      5      nUU-   nU[$        R&                  " UR                  S	   5      -  n[)        5       (       a  U R*                  OS n[-        UUUS
US9nO.[.        R1                  S5        [2        R4                  " UUUUSS
S9nUR	                  SS5      R7                  U
SUR                  U-  5      nUR9                  UR:                  5      nUR<                  S   " U5      nUR<                  S   " U5      nU	S:X  a$  UR	                  SS5      R7                  U
WWW5      nUR>                  (       a  X(-   nX!R@                  -  nU$ )Nr  r7   r	   rH   c              3   D   #    U  H  oR                   S    S:  v   M     g7f)r	   i   N)r   ).0r   s     rx   	<genexpr>4XLAFlashAttnProcessor2_0.__call__.<locals>.<genexpr>.  s     I5H6||A$&5Hs    r   r  r  r   F)causalr|   z[Unable to use the flash attention pallas kernel API call due to QKV sequence length < 4096.r  r  )!r^   r   r   r  r  r#  r   rK   rd   rb   r&  re   rf   r_   r`   allr  masked_fillr}  r~  r   r|   r   r   r   rr   rN   r   r   r   rk   r.   r-   )ru   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r   r   r  r  rR   r  r|   s                         rx   r   !XLAFlashAttnProcessor2_0.__call__
  s    !( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C IeS%5HIII)!/!4!4ZA~G[G[\^G_!` #((* [1!4eFmD [1!4eCjA  N*TYYu{{1~..E4;IIT004N+E3e\jkMNNm ::sE^sV[M &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   )r|   r9  r  )r   r;  r<  r=  r>  r   r   rA  rP   r   r   rB  r   rD  r  rz   rx   r   r   
  s    	-xhsmS6H0I'J 	- 9=15'+aa ||a  (5	a
 !.a u||$a 
a arz   r   c                       \ rS rSrSrS r  SS\S\R                  S\	\R                     S\	\R                     S	\R                  4
S
 jjr
Srg)MochiVaeAttnProcessor2_0iX  z(
Attention processor used in Mochi VAE.
c                 D    [        [        S5      (       d  [        S5      eg r  r  r  s    rx   rP   !MochiVaeAttnProcessor2_0.__init__]  r  rz   Nr  r   r   r   r}   c           	      ^   UnUR                   S   S:H  nUc  UR                   OUR                   u  pxn	Ub<  UR                  XHU5      nUR                  XqR                  SUR                   S   5      nU(       a`  UR	                  U5      nUR
                  S   " U5      nUR
                  S   " U5      nUR                  (       a  X%-   nX!R                  -  nU$ UR                  U5      n
Uc  UnUR                  U5      nUR	                  U5      nUR                   S   nXR                  -  nU
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U
5      n
UR                  b  UR                  U5      n[        R                  " XXSUR                  S9nUR                  SS5      R!                  USUR                  U-  5      nUR#                  U
R$                  5      nUR
                  S   " U5      nUR
                  S   " U5      nUR                  (       a  X%-   nX!R                  -  nU$ )Nr7   rH   r   r	   r  r  )r   r#  r  r   rf   rk   r.   r-   rd   re   r  r_   r`   rr   rN   r5   r   r   r   )ru   r  r   r   r   r  is_single_framer   r  r   r   r  r  rR   r  s                  rx   r   !MochiVaeAttnProcessor2_0.__call__a  s{    !'--a0A5 $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN IIm4M !KKN=9M KKN=9M'' - 8),F,FFM  		-( ($1!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C 663RVR`R`
 &//15==j"djj[cNcd%((5 A}5A}5##)4M%(B(BBrz   r  r:  r  r  rz   rx   r  r  X  sp    r 9=15II ||I  (5	I
 !.I 
I Irz   r  c                      \ rS rSrSrS rS\R                  S\\R                     S\R                  4S jr	   SS	\
S
\R                  S\\R                     S\\R                     S\\R                     S\R                  4S jjrSrg)StableAudioAttnProcessor2_0i  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the Stable Audio model. It applies rotary embedding on query and key vector, and allows MHA, GQA or MQA.
c                 D    [        [        S5      (       d  [        S5      eg )NrN   z[StableAudioAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   $StableAudioAttnProcessor2_0.__init__  r  rz   r  	freqs_cisr}   c                     SSK Jn  US   R                  S   nUSS U24   USUS 24   peU" XRSSS9n[        R                  " Xv4SS	9nU$ )
Nr7   rS  r   rH   .Tr  use_realuse_real_unbind_dimr  )r8  r  r   r   r  )	ru   r  r  r  rot_dimx_to_rotatex_unrotated	x_rotatedouts	            rx   apply_partial_rotary_emb4StableAudioAttnProcessor2_0.apply_partial_rotary_emb  sa    
 	1A,$$R(#$S(7(]#3QsGH}5E[$[d`bc	ii0b9
rz   Nr  r   r   r   
rotary_embc           	      d   SSK Jn  UnUR                  nUS:X  a3  UR                  u  ppUR	                  XX-  5      R                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XMU	5      nUR	                  XR                  SUR                  S   5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   UR                  -  nUR                  S   U-  nUR	                  U	SUR                  U5      R                  SS5      nUR	                  U	SUU5      R                  SS5      nUR	                  U	SUU5      R                  SS5      nUUR                  :w  a]  UR                  U-  n[        R                  " UUSUR                  S   U-  S9n[        R                  " UUSUR                  S   U-  S9nUR                  b  UR                  U5      nUR                   b  UR!                  U5      nUb  UR"                  nUR"                  nUR%                  [        R&                  5      nUR%                  [        R&                  5      nUS   R                  S   nUSS U24   USUS 24   nnU" UUS	S
S9n[        R(                  " UU4SS9nUR*                  (       d2  USS U24   USUS 24   nnU" UUS	S
S9n[        R(                  " UU4SS9nUR%                  U5      nUR%                  U5      n[,        R.                  " UUUUSSS9nUR                  SS5      R1                  U	SUR                  U-  5      nUR%                  UR"                  5      nUR2                  S   " U5      nUR2                  S   " U5      nUS:X  a$  UR                  SS
5      R1                  U	W
WW5      nUR4                  (       a  X'-   nX!R6                  -  nU$ )Nr7   rS  r  r	   rH   r  r   .Tr  r  r  r  Fr  )r8  r  r   r   r  r  r#  r   rd   rb   r&  re   rf   r   r  r_   r`   r   r   rn  r  rU   rr   rN   r   rk   r.   r-   )ru   r  r   r   r   r  r  r  r  r   r  r  r  r  r   r   r  r  r  r   heads_per_kv_headquery_dtype	key_dtyper  query_to_rotatequery_unrotatedquery_rotatedkey_to_rotatekey_unrotatedkey_rotateds                                 rx   r   $StableAudioAttnProcessor2_0.__call__  s    	1 "''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN		-( ($1!__$($C$CDY$Z!ii-.		/0;;r?djj099R=H,

:r4::x@JJ1aPhhz2x:DDQJ

:r8X>HHANtzz! $

h 6))#/@aUXU^U^_`UaduUuvC++(aU[[^N_=_E ;;"KK&E;;"++c"C !++K		IHHU]]+E&&'C m))"-G/4S(7(]/CU3PWPX=EY_O,_jSWmopMII}o>BGE**/23=/A3sGH}CU}.}jSWmopiim <"EHH[)E&&#C 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r  r  )r   r;  r<  r=  r>  rP   r   rB  r   r  r   r   r   rD  r  rz   rx   r  r    s    
<< & 
	& 9=15-1gg ||g  (5	g
 !.g U\\*g 
g grz   r  c                       \ rS rSrSrS r    SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\	\R                     S\R                  4S jjr
Srg)HunyuanAttnProcessor2_0i2  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the HunyuanDiT model. It applies a s normalization layer and rotary embedding on query and key vector.
c                 D    [        [        S5      (       d  [        S5      eg r  r  r  s    rx   rP    HunyuanAttnProcessor2_0.__init__8  r  rz   Nr  r   r   r   r  r  r}   c           	         SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XNU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                   b  UR!                  U5      nUb#  U" UU5      nUR"                  (       d	  U" UU5      n[$        R&                  " UUUUSSS9nUR                  SS5      R)                  U
SUR                  U-  5      nUR+                  UR,                  5      nUR.                  S	   " U5      nUR.                  S   " U5      nU	S:X  a$  UR                  SS
5      R)                  U
WWW5      nUR0                  (       a  X(-   nX!R2                  -  nU$ Nr7   rS  r  r	   rH   r  Fr  r   r  )r8  r  r^   r   r   r  r  r#  r   rK   rd   rb   r&  re   rf   r_   r`   rU   rr   rN   r   r   r   rk   r.   r-   )ru   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r   r   r  r  rR   r  s                        rx   r    HunyuanAttnProcessor2_0.__call__<  s    	1 ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '$U,<=E**&s,<= 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r  r:  r  r  rz   rx   r  r  2  s    
r 9=15'+37UU ||U  (5	U
 !.U u||$U #5<<0U 
U Urz   r  c                       \ rS rSrSrS r    SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\	\R                     S\R                  4S jjr
Srg)FusedHunyuanAttnProcessor2_0i  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0) with fused
projection layers. This is used in the HunyuanDiT model. It applies a s normalization layer and rotary embedding on
query and key vector.
c                 D    [        [        S5      (       d  [        S5      eg )NrN   z\FusedHunyuanAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   %FusedHunyuanAttnProcessor2_0.__init__  s%    q899n  :rz   Nr  r   r   r   r  r  r}   c           	      <   SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUc  UR                  OUR                  u  pnUb<  UR                  XNU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUc>  UR                  U5      nUR                  S   S-  n[        R                  " UUSS9u  nnnOoUR                  (       a  UR                  U5      nUR                  U5      nUR!                  U5      nUR                  S   S-  n[        R                  " UUSS9u  nnUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR"                  b  UR#                  U5      nUR$                  b  UR%                  U5      nUb#  U" UU5      nUR&                  (       d	  U" UU5      n[(        R*                  " UUUUSS	S
9nUR                  SS5      R-                  U
SUR                  U-  5      nUR/                  UR0                  5      nUR2                  S   " U5      nUR2                  S   " U5      nU	S:X  a$  UR                  SS5      R-                  U
WWW5      nUR4                  (       a  X(-   nX!R6                  -  nU$ )Nr7   rS  r  r	   rH   r   r  r  Fr  r   r  )r8  r  r^   r   r   r  r  r#  r   rK   r*  r   r!  rb   r&  rd   r,  r_   r`   rU   rr   rN   r   r   r   rk   r.   r-   )ru   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r   r"  r#  r   r  r  kvrR   r  s                           rx   r   %FusedHunyuanAttnProcessor2_0.__call__  s_    	1 ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM (++m,C2!+J %C DE3(,(G(GH](^%IIm,E12B"*JR<JCIIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '$U,<=E**&s,<= 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r  r:  r  r  rz   rx   r  r    s     9=15'+37WW ||W  (5	W
 !.W u||$W #5<<0W 
W Wrz   r  c                       \ rS rSrSrS r    SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\	\R                     S\R                  4S jjr
Srg)PAGHunyuanAttnProcessor2_0i  O  
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the HunyuanDiT model. It applies a normalization layer and rotary embedding on query and key vector. This
variant of the processor employs [Pertubed Attention Guidance](https://huggingface.co/papers/2403.17377).
c                 D    [        [        S5      (       d  [        S5      eg )NrN   zZPAGHunyuanAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   #PAGHunyuanAttnProcessor2_0.__init__  s%    q899l  :rz   Nr  r   r   r   r  r  r}   c           	      f   SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                  S5      u  pUc  UR                  OUR                  u  n
nnUb=  UR                  UUU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                   b  UR!                  U5      nUR"                  b  UR#                  U5      nUb#  U" UU5      nUR$                  (       d	  U" UU5      n[&        R(                  " UUUUSSS9nUR                  SS5      R+                  U
SUR                  U-  5      nUR-                  UR.                  5      nUR0                  S	   " U5      nUR0                  S   " U5      nU	S:X  a$  UR                  SS
5      R+                  U
WWW5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR-                  UR.                  5      nUR0                  S	   " U5      nUR0                  S   " U5      nU	S:X  a$  UR                  SS
5      R+                  U
WWW5      n[2        R4                  " X/5      nUR6                  (       a  X(-   nX!R8                  -  nU$ r  )r8  r  r^   r   r   r  r  r  r#  r   rK   rd   rb   r&  re   rf   r_   r`   rU   rr   rN   r   r   r   rk   r   r  r.   r-   )ru   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r   r  r   r   r  r  rR   r  s                          rx   r   #PAGHunyuanAttnProcessor2_0.__call__  s    	1 ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM 0=/B/B1/E, (='D##J_JeJe 	'
OQ %!88ZdeN ,00ZZ^MaMabdMefN??& $0A0K0KAq0Q R \ \]^`a b		+, ($5!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '$U,<=E**&s,<= ::33RW
 .771=EEjRTVZV`V`ckVkl-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o ??& $0A0K0KAq0Q R \ \]^`a b II&78-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o 		#4"HI##)4M%(B(BBrz   r  r:  r  r  rz   rx   r  r    s     9=15'+37kk ||k  (5	k
 !.k u||$k #5<<0k 
k krz   r  c                       \ rS rSrSrS r    SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\	\R                     S\R                  4S jjr
Srg)PAGCFGHunyuanAttnProcessor2_0iv  r  c                 D    [        [        S5      (       d  [        S5      eg )NrN   z]PAGCFGHunyuanAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   &PAGCFGHunyuanAttnProcessor2_0.__init__}  s%    q899o  :rz   Nr  r   r   r   r  r  r}   c           	         SSK Jn  UnUR                  b  UR                  X%5      nUR                  n	U	S:X  a3  UR                  u  ppUR                  XX-  5      R                  SS5      nUR                  S5      u  pn[        R                  " X/5      nUc  UR                  OUR                  u  n
nnUb=  UR                  UUU
5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR!                  U5      nUR#                  U5      nUR                  S   nUUR                  -  nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR                  U
SUR                  U5      R                  SS5      nUR$                  b  UR%                  U5      nUR&                  b  UR'                  U5      nUb#  U" UU5      nUR(                  (       d	  U" UU5      n[*        R,                  " UUUUSSS	9nUR                  SS5      R/                  U
SUR                  U-  5      nUR1                  UR2                  5      nUR4                  S
   " U5      nUR4                  S   " U5      nU	S:X  a$  UR                  SS5      R/                  U
WWW5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUR#                  U5      nUR1                  UR2                  5      nUR4                  S
   " U5      nUR4                  S   " U5      nU	S:X  a$  UR                  SS5      R/                  U
WWW5      n[        R                  " UU/5      nUR6                  (       a  X(-   nX!R8                  -  nU$ )Nr7   rS  r  r	   r   rH   r  Fr  r   r  )r8  r  r^   r   r   r  r  r  r   r  r#  r   rK   rd   rb   r&  re   rf   r_   r`   rU   rr   rN   r   r   r   rk   r.   r-   )ru   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r  r   r  r   r   r  r  rR   r  s                           rx   r   &PAGCFGHunyuanAttnProcessor2_0.__call__  s"    	1 ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM FSEXEXYZE[B1B!II';&OP (='D##J_JeJe 	'
OQ %!88ZdeN ,00ZZ^MaMabdMefN??& $0A0K0KAq0Q R \ \]^`a b		+, ($5!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C '$U,<=E**&s,<= ::33RW
 .771=EEjRTVZV`V`ckVkl-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o ??& $0A0K0KAq0Q R \ \]^`a b II&78-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o 		#46G"HI##)4M%(B(BBrz   r  r:  r  r  rz   rx   r  r  v  s     9=15'+37ll ||l  (5	l
 !.l u||$l #5<<0l 
l lrz   r  c                       \ rS rSrSrS r    SS\S\R                  S\R                  S\	\R                     S	\	\R                     S
\	\R                     S\	\
   S\R                  4S jjrSrg)LuminaAttnProcessor2_0i  z
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). This is
used in the LuminaNextDiT model. It applies a s normalization layer and rotary embedding on query and key vector.
c                 D    [        [        S5      (       d  [        S5      eg r  r  r  s    rx   rP   LuminaAttnProcessor2_0.__init__  r  rz   Nr  r   r   r   query_rotary_embkey_rotary_embbase_sequence_lengthr}   c                    SSK Jn  UR                  n	U	S:X  a3  UR                  u  ppUR	                  XX-  5      R                  SS5      nUR                  u  pnUR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUR                  S   nUUR                  -  nUR                  nUU-  nUR                  b  UR                  U5      nUR                  b  UR                  U5      nUR	                  U
SUR                  U5      nUR	                  U
SUU5      nUR	                  U
SUU5      nUb  U" UUSS9nUb  U" UUSS9nUR                  U5      UR                  U5      nnUc  S nOGUb8  [        R                  " [        R                   " X5      5      UR"                  -  nOUR"                  nUR                  U-  nUS:  ah  UR%                  S5      R'                  SSSUS5      R)                  SS5      nUR%                  S5      R'                  SSSUS5      R)                  SS5      nUR+                  5       R	                  U
SSS5      nUR-                  SUR                  US5      nUR                  SS5      nUR                  SS5      nUR                  SS5      n[.        R0                  " UUUUUS	9nUR                  SS5      R                  U5      nU$ )
Nr7   rS  r  r	   rH   F)r  r   )r  rY   )r8  r  r   r   r  r  rd   re   rf   r   r   r_   r`   r   r}  r~  logrY   r  rw  r  r@  rf  rr   rN   )ru   r  r   r   r   r  r  r  r  r  r   r  r  r  r  r   r   r  r  r   rR   r  r   r   softmax_scalen_reps                             rx   r   LuminaAttnProcessor2_0.__call__  s    	1"''
?1>1D1D.J)..zFNS]]^_abcM)6)<)<&
Q 		-(ii-.		/0KKO	IIbM	

* ( ;;"KK&E;;"++c"C

:r4::x@hhz2x:

:r8X> '$U,<uME%"3GCXXe_cffUms ! M#/ $		$((?*Y Z]a]g]g g $

 

h&A:--"))!Q5!<DDQJCOOA&--aAua@HHANE (,,.33J1bI'..r4::PRS1%mmAq!1% 663}
 &//1588?rz   r  r:  )r   r;  r<  r=  r>  rP   r   r   rB  r   r?  r   rD  r  rz   rx   r  r    s    
r 263715.2SS ||S  %||	S
 !.S #5<<0S !.S 'smS 
S Srz   r  c                       \ rS rSrSrS r   SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\R                  4S jjr
Srg)FusedAttnProcessor2_0iR  u  
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). It uses
fused projection layers. For self-attention modules, all projection matrices (i.e., query, key, value) are fused.
For cross-attention modules, key and value projection matrices are fused.

<Tip warning={true}>

This API is currently 🧪 experimental in nature and can change in future.

</Tip>
c                 D    [        [        S5      (       d  [        S5      eg )NrN   z`FusedAttnProcessor2_0 requires at least PyTorch 2.0, to use it. Please upgrade PyTorch to > 2.0.r  r  s    rx   rP   FusedAttnProcessor2_0.__init___  s%    q899r  :rz   Nr  r   r   r   r  r}   c           	      F   [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        Un	UR                  b  UR                  X%5      nUR                  n
U
S:X  a3  UR
                  u  ppUR                  XX-  5      R                  SS5      nUc  UR
                  OUR
                  u  pnUb<  UR                  XOU5      nUR                  XR                  SUR
                  S   5      nUR                  b1  UR                  UR                  SS5      5      R                  SS5      nUc>  UR                  U5      nUR
                  S   S	-  n[        R                  " UUSS
9u  nnnOoUR                  (       a  UR                  U5      nUR!                  U5      nUR#                  U5      nUR
                  S   S-  n[        R                  " UUSS
9u  nnUR
                  S   nUUR                  -  nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR$                  b  UR%                  U5      nUR&                  b  UR'                  U5      n[(        R*                  " UUUUSSS9nUR                  SS5      R-                  USUR                  U-  5      nUR/                  UR0                  5      nUR2                  S   " U5      nUR2                  S   " U5      nU
S:X  a$  UR                  SS5      R-                  UWWW5      nUR4                  (       a  X)-   nX!R6                  -  nU$ )Nr   rY   r  r  r  r7   r	   rH   r   r  r  Fr  r  )r   r  r   r^   r   r   r  r  r#  r   rK   r*  r   r!  rb   r&  rd   r,  r_   r`   rr   rN   r   r   r   rk   r.   r-   )ru   r  r   r   r   r  r  r  r  r  r  r   r  r  r  r  r   r"  r#  r   r  r  r  rR   r  s                            rx   r   FusedAttnProcessor2_0.__call__e  sb    t9q=FJJw5A #Ugw(;< ( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM (++m,C2!+J %C DE3(,(G(GH](^%IIm,E12B"*JR<JCIIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP;;"KK&E;;"++c"C 6633RW
 &//15==j"djj[cNcd%((5 A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r  r  r  r  rz   rx   r  r  R  s    
 9=15'+TT ||T  (5	T
 !.T u||$T 
T Trz   r  c                      ^  \ rS rSrSr       SS\S\S\\   S\\   S\S\S	\\	   4U 4S
 jjjr
  SS\S\R                  S\\R                     S\\R                     S\R                  4
S jjrSrU =r$ )r   i  a.  
Processor for implementing memory efficient attention using xFormers for the Custom Diffusion method.

Args:
train_kv (`bool`, defaults to `True`):
    Whether to newly train the key and value matrices corresponding to the text features.
train_q_out (`bool`, defaults to `True`):
    Whether to newly train query matrices corresponding to the latent image features.
hidden_size (`int`, *optional*, defaults to `None`):
    The hidden size of the attention layer.
cross_attention_dim (`int`, *optional*, defaults to `None`):
    The number of channels in the `encoder_hidden_states`.
out_bias (`bool`, defaults to `True`):
    Whether to include the bias parameter in `train_q_out`.
dropout (`float`, *optional*, defaults to 0.0):
    The dropout probability to use.
attention_op (`Callable`, *optional*, defaults to `None`):
    The base
    [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to use
    as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best operator.
r   r   r   r   r)   r   r   c                 X  > [         TU ]  5         Xl        X l        X0l        X@l        Xpl        U R                  (       aH  [        R                  " U=(       d    UUSS9U l	        [        R                  " U=(       d    UUSS9U l
        U R                  (       a  [        R                  " X3SS9U l        [        R                  " / 5      U l        U R                  R                  [        R                  " X3US95        U R                  R                  [        R                  " U5      5        g g r  )rO   rP   r   r   r   r   r   r   rc   r   r  r  rj   r  rl   rm   )	ru   r   r   r   r   r)   r   r   rw   s	           rx   rP   -CustomDiffusionXFormersAttnProcessor.__init__  s     	 &&#6 ( ==)+3F3U+Wbin)oD&)+3F3U+Wbin)oD&)+;RW)XD&+-==+<D(((//		+Ya0bc((//

70CD	 rz   r  r   r   r   r}   c           	         Uc  UR                   OUR                   u  pVnUR                  XFU5      nU R                  (       a?  U R                  U5      R	                  UR
                  R                  R                  5      nO>UR                  UR	                  UR
                  R                  R                  5      5      nUc  Sn	UnO$Sn	UR                  (       a  UR                  U5      nU R                  (       a  U R                  UR	                  U R                  R                  R                  5      5      n
U R                  UR	                  U R                  R                  R                  5      5      nU
R	                  UR
                  R                  R                  5      n
UR	                  UR
                  R                  R                  5      nO"UR                  U5      n
UR                  U5      nU	(       ai  [        R                   " U
5      nUS S 2S S2S S 24   S-  US S 2S S2S S 24'   X-  SU-
  U
R#                  5       -  -   n
X-  SU-
  UR#                  5       -  -   nUR%                  U5      R'                  5       nUR%                  U
5      R'                  5       n
UR%                  U5      R'                  5       n[(        R*                  R-                  XXU R.                  UR0                  S9nUR	                  UR                  5      nUR3                  U5      nU R                  (       a,  U R4                  S   " U5      nU R4                  S   " U5      nU$ UR6                  S   " U5      nUR6                  S   " U5      nU$ )NFTr7   r  r-  r   )r   r#  r   r  r   rd   r   r   rb   r&  r   r   r  re   rf   r   r  r  r   r0  r   r   r   r   rY   r   r  rk   )ru   r  r   r   r   r   r  r   r   r  r  r  r  s                rx   r   -CustomDiffusionXFormersAttnProcessor.__call__  s    $9#@MF[FaFa 	'
Q 44^V`a..}=@@AQAQAWAWXEIIm..tyy/?/?/E/EFGE (I$1!I(,(G(GH](^%==,,-B-E-EdF`F`FgFgFmFm-noC../D/G/GHbHbHiHiHoHo/pqE&&))//0CHHTYY--334E))12CII34E__S)F%a!Qh/#5F1bqb!8,!f*

!<<CNa&jELLN%BBE&&u-88:$$S)446&&u-88: ??D<M<MUYU_U_ @ 
 &((5..}= 88;MJM 88;MJM 	 !KKN=9M KKN=9Mrz   )	r   r   r   r   r  r  r  r   r   )TFNNTr  Nr:  )r   r;  r<  r=  r>  r@  r   r?  r  r   rP   r   r   rB  r   rD  rE  rF  s   @rx   r   r     s    0 !%)-1+/EE E c]	E
 &c]E E E x(E E@ 9=15>> ||>  (5	>
 !.> 
> >rz   r   c                      ^  \ rS rSrSr      SS\S\S\\   S\\   S\S\4U 4S	 jjjr	  SS
\
S\R                  S\\R                     S\\R                     S\R                  4
S jjrSrU =r$ )r   i0  uS  
Processor for implementing attention for the Custom Diffusion method using PyTorch 2.0’s memory-efficient scaled
dot-product attention.

Args:
    train_kv (`bool`, defaults to `True`):
        Whether to newly train the key and value matrices corresponding to the text features.
    train_q_out (`bool`, defaults to `True`):
        Whether to newly train query matrices corresponding to the latent image features.
    hidden_size (`int`, *optional*, defaults to `None`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`, *optional*, defaults to `None`):
        The number of channels in the `encoder_hidden_states`.
    out_bias (`bool`, defaults to `True`):
        Whether to include the bias parameter in `train_q_out`.
    dropout (`float`, *optional*, defaults to 0.0):
        The dropout probability to use.
r   r   r   r   r)   r   c                 L  > [         TU ]  5         Xl        X l        X0l        X@l        U R                  (       aH  [        R                  " U=(       d    UUSS9U l        [        R                  " U=(       d    UUSS9U l	        U R                  (       a  [        R                  " X3SS9U l
        [        R                  " / 5      U l        U R                  R                  [        R                  " X3US95        U R                  R                  [        R                  " U5      5        g g r  r  r  s          rx   rP   (CustomDiffusionAttnProcessor2_0.__init__D  r  rz   r  r   r   r   r}   c           	         UR                   u  pVnUR                  XFU5      nU R                  (       a  U R                  U5      nOUR	                  U5      nUc  Sn	UnO$Sn	UR
                  (       a  UR                  U5      nU R                  (       a  U R                  UR                  U R                  R                  R                  5      5      n
U R                  UR                  U R                  R                  R                  5      5      nU
R                  UR                  R                  R                  5      n
UR                  UR                  R                  R                  5      nO"UR                  U5      n
UR                  U5      nU	(       ai  [        R                   " U
5      nUS S 2S S2S S 24   S-  US S 2S S2S S 24'   X-  SU-
  U
R#                  5       -  -   n
X-  SU-
  UR#                  5       -  -   nUR                   S   nXR$                  -  nUR'                  USUR$                  U5      R)                  SS5      nU
R'                  USUR$                  U5      R)                  SS5      n
UR'                  USUR$                  U5      R)                  SS5      n[*        R,                  " XXSSS9nUR)                  SS5      R/                  USUR$                  U-  5      nUR                  UR                  5      nU R                  (       a,  U R0                  S   " U5      nU R0                  S   " U5      nU$ UR2                  S   " U5      nUR2                  S   " U5      nU$ )	NFTr7   r  rH   r	   r  r   )r   r#  r   r  rd   rb   r&  r   r   r   r   r   r  re   rf   r   r  r  r   r  r  rr   rN   r   r  rk   )ru   r  r   r   r   r   r  r   r   r  r  r  r  rR   r  s                  rx   r   (CustomDiffusionAttnProcessor2_0.__call__^  s    *7)<)<&
Q44^V`a..}=EIIm,E (I$1!I(,(G(GH](^%==,,-B-E-EdF`F`FgFgFmFm-noC../D/G/GHbHbHiHiHoHo/pqE&&))//0CHHTYY--334E ))12CII34E__S)F%a!Qh/#5F1bqb!8,!f*

!<<CNa&jELLN%BBE!''+	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP 663RW
 &//15==j"djj[cNcd%((5 88;MJM 88;MJM 	 !KKN=9M KKN=9Mrz   r  r  r:  r  rF  s   @rx   r   r   0  s    *  %)-1EE E c]	E
 &c]E E E E< 9=15AA ||A  (5	A
 !.A 
A Arz   r   c                       \ rS rSrSrS\4S jr  SS\S\R                  S\
\R                     S	\
\R                     S
\R                  4
S jjrSrg)r   i  a  
Processor for implementing sliced attention.

Args:
    slice_size (`int`, *optional*):
        The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and
        `attention_head_dim` must be a multiple of the `slice_size`.
r   c                     Xl         g r9  r   ru   r   s     rx   rP   SlicedAttnProcessor.__init__      $rz   Nr  r   r   r   r}   c                 d   UnUR                   nUS:X  a3  UR                  u  pxpUR                  XxX-  5      R                  SS5      nUc  UR                  OUR                  u  p{nUR	                  XKU5      nUR
                  b1  UR                  UR                  SS5      5      R                  SS5      nUR                  U5      nUR                  S   nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  u  nnn[        R                  " UUXR                  -  4UR                  UR                   S9n[#        US-
  U R$                  -  S-   5       Hj  nUU R$                  -  nUS-   U R$                  -  nUUU nUUU nUb  UUU OS nUR'                  UUU5      n[        R(                  " UUUU 5      nUUUU& Ml     UR+                  U5      nUR,                  S   " U5      nUR,                  S   " U5      nUS:X  a$  UR                  SS5      R/                  UWW	W
5      nUR0                  (       a  X%-   nX!R2                  -  nU$ )Nr  r7   r	   rH   r   r   r  )r   r   r  r  r#  rK   rd   r   rb   r&  re   rf   r   r  r   r   r   r  r   r  r  r   rk   r   r.   r-   )ru   r  r   r   r   r  r  r   r  r  r  r  r   r   rJ   r  r  batch_size_attentionrh  i	start_idxend_idxquery_slice	key_sliceattn_mask_slice
attn_slices                             rx   r   SlicedAttnProcessor.__call__  s    !"''
?1>1D1D.J)..zFNS]]^_abcM $9#@MF[FaFa 	'
Q 44^V`a??& OOM,C,CAq,IJTTUVXYZM		-(kk"o&&u- ($1!__$($C$CDY$Z!ii-.		/0$$S)&&u-05-lA!<

1BCELL`e`k`k
 ,q0T__DqHIADOO+I1u/G	'2KIg.ICQC]nYw?cgO22;	?[J:uYw/GHJ/9M)G, J ..}= A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r  r:  )r   r;  r<  r=  r>  r?  rP   r   r   rB  r   r   rD  r  rz   rx   r   r     sv    %3 % 9=15GG ||G  (5	G
 !.G 
G Grz   r   c                       \ rS rSrSrS r   SSSS\R                  S\\R                     S	\\R                     S
\\R                     S\R                  4S jjr	Sr
g)r   i  aP  
Processor for implementing sliced attention with extra learnable key and value matrices for the text encoder.

Args:
    slice_size (`int`, *optional*):
        The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and
        `attention_head_dim` must be a multiple of the `slice_size`.
c                     Xl         g r9  r  r  s     rx   rP   #SlicedAttnAddedKVProcessor.__init__  r  rz   Nr  r   r   r   r   r  r}   c                 (   UnUR                   b  UR                  X%5      nUR                  UR                  S   UR                  S   S5      R                  SS5      nUR                  u  pxn	UR	                  XHU5      nUc  UnO"UR
                  (       a  UR                  U5      nUR                  UR                  SS5      5      R                  SS5      nUR                  U5      n
U
R                  S   nUR                  U
5      n
UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  (       dq  UR                  U5      nUR                  U5      nUR                  U5      nUR                  U5      n[        R                   " X/SS9n[        R                   " X/SS9nOUnUnU
R                  u  nnn	[        R"                  " UUXR$                  -  4U
R&                  U
R(                  S9n[+        US-
  U R,                  -  S-   5       Hj  nUU R,                  -  nUS-   U R,                  -  nU
UU nUUU nUb  UUU OS nUR/                  UUU5      n[        R0                  " UUUU 5      nUUUU& Ml     UR3                  U5      nUR4                  S   " U5      nUR4                  S   " U5      nUR                  SS5      R7                  UR                  5      nX&-   nU$ )Nr   r7   rH   r	   r  r   r  )r^   r  r   r  r#  rb   r&  rK   rd   r   rg   rh   r+   re   rf   r   r  r  r   r   r   r  r   r  r  r   rk   r   )ru   r  r   r   r   r  r  r   r  r   r   rJ   r  r  r  r  r  rh  r  r  r  r  r  r  r  s                            rx   r   #SlicedAttnAddedKVProcessor.__call__  s    !( --mBM%**=+>+>q+A=CVCVWXCY[]^hhijlmn)6)<)<&
Q44^V`a ($1!__$($C$CDY$Z!(?(?1(EFPPQRTUV		-(kk"o&&u-)-9N)O&+/??;P+Q()-)?)?@^)_&+/+A+ABb+c((())M*CIIm,E((-C**51E));AqICII?GQOE0C4E05-lA!<

1BCELL`e`k`k
 ,q0T__DqHIADOO+I1u/G	'2KIg.ICQC]nYw?cgO22;	?[J:uYw/GHJ/9M)G, J ..}= A}5A}5%//B7??O%0rz   r  r  r  r  rz   rx   r   r     s    % 9=15'+LL ||L  (5	L
 !.L u||$L 
L Lrz   r   c                      ^  \ rS rSrSrS\S\4U 4S jjrS\R                  S\R                  S\R                  4S	 jr	S
r
U =r$ )r]   iU  aU  
Spatially conditioned normalization as defined in https://huggingface.co/papers/2209.09002.

Args:
    f_channels (`int`):
        The number of channels for input to group normalization layer, and output of the spatial norm layer.
    zq_channels (`int`):
        The number of channels for the quantized vector as described in the paper.
r@   rA   c                    > [         TU ]  5         [        R                  " USSSS9U l        [        R
                  " X!SSSS9U l        [        R
                  " X!SSSS9U l        g )Nr8  ư>Tr<   r7   r   )rK  strider"  )rO   rP   r   r\   
norm_layerrN  conv_yconv_b)ru   r@   rA   rw   s      rx   rP   SpatialNorm.__init__`  sV    
 	,,J2SW`deiiQqZ[\iiQqZ[\rz   fzqr}   c                     UR                   SS  n[        R                  " X#SS9nU R                  U5      nX@R	                  U5      -  U R                  U5      -   nU$ )Nr  nearest)r  rk  )r   rr   interpolater	  r
  r  )ru   r  r  f_sizenorm_fnew_fs         rx   r   SpatialNorm.forwardj  sQ    ]]2;#R(4;;r?:rz   )r  r
  r	  )r   r;  r<  r=  r>  r?  rP   r   rB  r   rD  rE  rF  s   @rx   r]   r]   U  sM    ]] ] 5<< ELL  rz   r]   c                      ^  \ rS rSrSrSU 4S jjr     SS\S\R                  S\	\R                     S\	\R                     S\	\R                     S	\
S
\	\R                     4S jjrSrU =r$ )r   ir  a  
Attention processor for Multiple IP-Adapters.

Args:
    hidden_size (`int`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`):
        The number of channels in the `encoder_hidden_states`.
    num_tokens (`int`, `Tuple[int]` or `List[int]`, defaults to `(4,)`):
        The context length of the image features.
    scale (`float` or List[`float`], defaults to 1.0):
        the weight scale of image prompt.
c                 t  > [         TU ]  5         Xl        X l        [	        U[
        [        45      (       d  U/nX0l        [	        U[        5      (       d  U/[        U5      -  n[        U5      [        U5      :w  a  [        S5      eX@l
        [        R                  " [        [        U5      5       Vs/ s H  n[        R                  " X!SS9PM     sn5      U l        [        R                  " [        [        U5      5       Vs/ s H  n[        R                  " X!SS9PM     sn5      U l        g s  snf s  snf NJ`scale` should be a list of integers with the same length as `num_tokens`.FrM   )rO   rP   r   r   r   tuplelistr   r   r[   rY   r   rj   r  rc   r   to_v_ipru   r   r   r   rY   r   rw   s         rx   rP   IPAdapterAttnProcessor.__init__  s    &#6 *udm44$J$%&&Gc*o-Eu:Z(ijj
}}NSTWXbTcNdeNdRYY*eDNde
 }}NSTWXbTcNdeNdRYY*eDNde
 f fs   3D0D5r  r   r   r   r  rY   r   c           
         UnUbc  [        U[        5      (       a  Uu  p9OISn
[        SSU
SS9  UR                  S   U R                  S   -
  nUS S 2S U2S S 24   US S 2US 2S S 24   /pUR
                  b  UR                  X%5      nUR                  nUS:X  a5  UR                  u  pnnUR                  XUU-  5      R                  SS	5      nUc  UR                  OUR                  u  nnnUR                  UUU5      nUR                  b1  UR                  UR                  SS	5      5      R                  SS	5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR                  U5      nUR!                  U5      nUR!                  U5      nUR!                  U5      nUR#                  UUU5      n[$        R&                  " UU5      nUR)                  U5      nUGb  [        U[*        5      (       d  [-        UR/                  S5      5      n[1        U5      [1        U R2                  5      s=:X  a  [1        W	5      :X  d<  O  [5        S
[1        U5       S[1        U R2                  5       S[1        W	5       S35      e[7        [9        XpR2                  U	5      5       H  u  nu  nnnUc  M  [        U[$        R:                  5      (       a  UR                  S:w  a  [5        S5      eUR                  S   UR                  S   :w  a.  [5        SUR                  S    SUR                  S    SU 35      e[        U[,        5      (       d  M  [1        U5      UR                  S   :X  a  M  [5        SUR                  S    S[1        U5       SU 35      e   OS /[1        U R2                  5      -  n[9        W	U R2                  U R<                  U R>                  U5       GH  u  nnnnnSn[        U[,        5      (       a  [A        S U 5       5      (       a  SnOUS:X  a  SnU(       a  ML  UGbD  [        U[,        5      (       d  U/UR                  S   -  nUR                  S   n[C        U5       H  nU" US S 2US S 2S S 24   5      n U" US S 2US S 2S S 24   5      n!UR!                  U 5      n UR!                  U!5      n!UR#                  UU S 5      n"[$        R&                  " U"U!5      n#UR)                  U#5      n#[D        RF                  " US S 2US S 2S S 24   UU#R                  S   U#R                  S	   5      n$U$RI                  URJ                  URL                  S9n$X&U   U#U$-  -  -   nM     GM  U" U5      n U" U5      n!UR!                  U 5      n UR!                  U!5      n!UR#                  UU S 5      n"[$        R&                  " U"U!5      nUR)                  U5      nX&U-  -   nGM     URN                  S   " U5      nURN                  S   " U5      nUS:X  a$  UR                  SS5      RQ                  UWWW5      nURR                  (       a  X(-   nX!RT                  -  nU$ )NYou have passed a tensor as `encoder_hidden_states`. This is deprecated and will be removed in a future release. Please make sure to update your script to pass `encoder_hidden_states` as a tuple to suppress this warning.!encoder_hidden_states not a tupler  Fstandard_warnr7   r   r  r	   "Length of ip_adapter_masks array ()) must match length of self.scale array (") and number of ip_hidden_states ()Each element of the ip_adapter_masks array should be a tensor with shape [1, num_images_for_ip_adapter, height, width]. Please use `IPAdapterMaskProcessor` to preprocess your maskNumber of masks (&) does not match number of ip images () at index #) does not match number of scales (c              3   *   #    U  H	  oS :H  v   M     g7fr   Nr  r  ss     rx   r  2IPAdapterAttnProcessor.__call__.<locals>.<genexpr>       -u!Avu   Tr  rH   r  )+r   r  r   r   r   r^   r   r  r  r#  rK   rd   rb   r&  re   rf   r   r  r   r  r   r   r  r  r   rY   r[   	enumerateziprB  r   r  r  r  r
   
downsampler   r   r   rk   r   r.   r-   )%ru   r  r   r   r   r  rY   r   r  r   r  end_posr  r   r  r  r  r  r   r   r  r  r  indexr  ip_statecurrent_ip_hidden_statesr   r  skipcurrent_num_imagesr  ip_keyip_valueip_attention_probs_current_ip_hidden_statesmask_downsamples%                                        rx   r   IPAdapterAttnProcessor.__call__  s    ! !,/77:O7%'7C $ =wH[kpq/55a84??1;MM)!XgXq.9*1gh>:; (8
 ( --mBM"''
?1>1D1D.J)..zFUNS]]^_abcM $9#@MF[FaFa 	'
OQ 44^_V`a??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0&&u-$$S)&&u-33E3O		/59..}='.55#'(8(B(B1(E#F ()S_UDT@UU 8=M9N8O P336tzz?2C D,-.a1  7@DTV`V`br@s6t2E2D%| %dELL99TYY!^([ 
 zz!}q(99(/

1 ?44<NN14E3FkRWQXZ  "%..s5zTZZPQ]7R(/

1 ?114UKwP  7u( !%vDJJ7 HKdjj$,,FVH
C$eWgt D%&&-u---D!4#%eT22!&$**Q- 7)-A&"#56!()A!Q1*)M!N#*+CAq!QJ+O#P!%!7!7!?#'#9#9(#C-1-F-FufVZ-[*49II>PRZ4[1484J4JKd4e1*@*K*K Aq!,&5;;A>5;;A>	+ +:*<*<5;;W\WcWc*<*d(5aD]`oDo8p(p) 7, %%=>F&'?@H!33F;F#55h?H)-)B)B5&RV)W&/4yy9KX/V,/3/E/EF^/_,$1<T4T$TM_H
d A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   r   r   r   rY   r   r  Nr  r;   NNNr;   Nr   r;  r<  r=  r>  rP   r   r   rB  r   r  r   rD  rE  rF  s   @rx   r   r   r  s    
6 9=15'+37__ ||_  (5	_
 !._ u||$_ _ #5<<0_ _rz   r   c                      ^  \ rS rSrSrSU 4S jjr     SS\S\R                  S\	\R                     S\	\R                     S\	\R                     S	\
S
\	\R                     4S jjrSrU =r$ )r   i:  a  
Attention processor for IP-Adapter for PyTorch 2.0.

Args:
    hidden_size (`int`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`):
        The number of channels in the `encoder_hidden_states`.
    num_tokens (`int`, `Tuple[int]` or `List[int]`, defaults to `(4,)`):
        The context length of the image features.
    scale (`float` or `List[float]`, defaults to 1.0):
        the weight scale of image prompt.
c                   > [         TU ]  5         [        [        S5      (       d"  [	        U R
                  R                   S35      eXl        X l        [        U[        [        45      (       d  U/nX0l        [        U[        5      (       d  U/[        U5      -  n[        U5      [        U5      :w  a  [        S5      eX@l        [         R"                  " [%        [        U5      5       Vs/ s H  n[         R&                  " X!SS9PM     sn5      U l        [         R"                  " [%        [        U5      5       Vs/ s H  n[         R&                  " X!SS9PM     sn5      U l        g s  snf s  snf )NrN   z@ requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  FrM   )rO   rP   rq   rr   r  rw   r   r   r   r   r  r  r   r   r[   rY   r   rj   r  rc   r   r  r  s         rx   rP   "IPAdapterAttnProcessor2_0.__init__I  s.   q899>>**++kl  '#6 *udm44$J$%&&Gc*o-Eu:Z(ijj
}}NSTWXbTcNdeNdRYY*eDNde
 }}NSTWXbTcNdeNdRYY*eDNde
 f fs   *E';E,r  r   r   r   r  rY   r   c                 >   UnUbc  [        U[        5      (       a  Uu  p9OISn
[        SSU
SS9  UR                  S   U R                  S   -
  nUS S 2S U2S S 24   US S 2US 2S S 24   /pUR
                  b  UR                  X%5      nUR                  nUS:X  a5  UR                  u  pnnUR                  XUU-  5      R                  SS	5      nUc  UR                  OUR                  u  nnnUb=  UR                  UUU5      nUR                  XR                  S
UR                  S
   5      nUR                  b1  UR                  UR                  SS	5      5      R                  SS	5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR!                  U5      nUR                  S
   nUUR                  -  nUR                  US
UR                  U5      R                  SS	5      nUR                  US
UR                  U5      R                  SS	5      nUR                  US
UR                  U5      R                  SS	5      n["        R$                  " UUUUSSS9nUR                  SS	5      R'                  US
UR                  U-  5      nUR)                  UR*                  5      nUGb  [        U[,        5      (       d  [/        UR1                  S5      5      n[3        U5      [3        U R4                  5      s=:X  a  [3        W	5      :X  d<  O  [7        S[3        U5       S[3        U R4                  5       S[3        W	5       S35      e[9        [;        XpR4                  U	5      5       H  u  nu  nnnUc  M  [        U[<        R>                  5      (       a  UR                  S:w  a  [7        S5      eUR                  S   UR                  S   :w  a.  [7        SUR                  S    SUR                  S    SU 35      e[        U[.        5      (       d  M  [3        U5      UR                  S   :X  a  M  [7        SUR                  S    S[3        U5       SU 35      e   OS /[3        U R4                  5      -  n[;        W	U R4                  U R@                  U RB                  U5       GH  u  nnnnnSn[        U[.        5      (       a  [E        S U 5       5      (       a  SnOUS:X  a  SnU(       a  ML  UGb  [        U[.        5      (       d  U/UR                  S   -  nUR                  S   n[G        U5       GH_  n U" US S 2U S S 2S S 24   5      n!U" US S 2U S S 2S S 24   5      n"U!R                  US
UR                  U5      R                  SS	5      n!U"R                  US
UR                  U5      R                  SS	5      n"["        R$                  " UU!U"S SSS9n#U#R                  SS	5      R'                  US
UR                  U-  5      n#U#R)                  UR*                  5      n#[H        RJ                  " US S 2U S S 2S S 24   UU#R                  S   U#R                  S	   5      n$U$R)                  UR*                  URL                  S9n$X&U    U#U$-  -  -   nGMb     GM  U" U5      n!U" U5      n"U!R                  US
UR                  U5      R                  SS	5      n!U"R                  US
UR                  U5      R                  SS	5      n"["        R$                  " UU!U"S SSS9nUR                  SS	5      R'                  US
UR                  U-  5      nUR)                  UR*                  5      nX&U-  -   nGM     URN                  S   " U5      nURN                  S   " U5      nUS:X  a$  UR                  S
S5      R'                  UWWW5      nURP                  (       a  X(-   nX!RR                  -  nU$ )Nr   r!  r  Fr"  r7   r   r  r	   rH   r  r  r$  r%  r&  r'  r(  r)  r*  r+  r,  c              3   *   #    U  H	  oS :H  v   M     g7fr.  r  r/  s     rx   r  5IPAdapterAttnProcessor2_0.__call__.<locals>.<genexpr>  r2  r3  Tr  r  )*r   r  r   r   r   r^   r   r  r  r#  r   rK   rd   rb   r&  re   rf   rr   rN   r   r   r   r   r  r  r   rY   r[   r4  r5  r   rB  r   r  r  r  r
   r6  r   rk   r.   r-   )%ru   r  r   r   r   r  rY   r   r  r   r  r7  r  r   r  r  r  r  r   r   r  r  rR   r  r8  r  r9  r:  r   r  r;  r<  r  r=  r>  r@  rA  s%                                        rx   r   "IPAdapterAttnProcessor2_0.__call__e  s
    ! !,/77:O7%'7C $ =wH[kpq/55a84??1;MM)!XgXq.9*1gh>:; (8
 ( --mBM"''
?1>1D1D.J)..zFUNS]]^_abcM $9#@MF[FaFa 	'
OQ %!88ZdeN ,00ZZ^MaMabdMefN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP 6633RW
 &//15==j"djj[cNcd%((5'.55#'(8(B(B1(E#F ()S_UDT@UU 8=M9N8O P336tzz?2C D,-.a1  7@DTV`V`br@s6t2E2D%| %dELL99TYY!^([ 
 zz!}q(99(/

1 ?44<NN14E3FkRWQXZ  "%..s5zTZZPQ]7R(/

1 ?114UKwP  7u( !%vDJJ7 HKdjj$,,FVH
C$eWgt D%&&-u---D!4#%eT22!&$**Q- 7)-A&"#56!()A!Q1*)M!N#*+CAq!QJ+O#P!'ZTZZ!R!\!\]^`a!b#+==RX#V#`#`abde#f 564R4R!68ts^c51 5N4W4WXY[\4]4e4e&DJJ,A51 5N4P4PQVQ\Q\4]1*@*K*K Aq!,&5;;A>5;;A>	+ +:*<*<5;;W\WcWc*<*d(5aD]`oDo8p(p5 78 %%=>F&'?@H#[[RXNXXYZ\]^F'}}ZTZZR\\]^`abH 01/M/Mvx43Z_0, 0H/Q/QRSUV/W/_/_"B

X(=0, 0H/J/J5;;/W,$1<T4T$TMyH
~ A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   rC  rD  rF  rG  rF  s   @rx   r   r   :  s    
@ 9=15'+37zz ||z  (5	z
 !.z u||$z z #5<<0z zrz   r   c                      ^  \ rS rSrSr    SS\\   4U 4S jjjr     SS\S\	R                  S\\	R                     S\\	R                     S	\\	R                     S
\S\\	R                     4S jjrSrU =r$ )r   i"  a  
Attention processor for IP-Adapter using xFormers.

Args:
    hidden_size (`int`):
        The hidden size of the attention layer.
    cross_attention_dim (`int`):
        The number of channels in the `encoder_hidden_states`.
    num_tokens (`int`, `Tuple[int]` or `List[int]`, defaults to `(4,)`):
        The context length of the image features.
    scale (`float` or `List[float]`, defaults to 1.0):
        the weight scale of image prompt.
    attention_op (`Callable`, *optional*, defaults to `None`):
        The base
        [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
        use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
        operator.
r   c                   > [         TU ]  5         Xl        X l        XPl        [        U[        [        45      (       d  U/nX0l        [        U[        5      (       d  U/[        U5      -  n[        U5      [        U5      :w  a  [        S5      eX@l        [        R                  " [        [        U5      5       Vs/ s H"  n[        R                  " U=(       d    UUSS9PM$     sn5      U l        [        R                  " [        [        U5      5       Vs/ s H"  n[        R                  " U=(       d    UUSS9PM$     sn5      U l        g s  snf s  snf r  )rO   rP   r   r   r   r   r  r  r   r   r[   rY   r   rj   r  rc   r   r  )ru   r   r   r   rY   r   r   rw   s          rx   rP   'IPAdapterXFormersAttnProcessor.__init__6  s    	&#6 (*udm44$J$%&&Gc*o-Eu:Z(ijj
}}]bcfgqcr]st]sXYRYY*9k;US]st
 }}]bcfgqcr]st]sXYRYY*9k;US]st
 u us   9)E
)Er  r   r   r   r  rY   r   c           
         UnUbc  [        U[        5      (       a  Uu  p9OISn
[        SSU
SS9  UR                  S   U R                  S   -
  nUS S 2S U2S S 24   US S 2US 2S S 24   /pUR
                  b  UR                  X%5      nUR                  nUS:X  a5  UR                  u  pnnUR                  XUU-  5      R                  SS	5      nUc  UR                  OUR                  u  nnnUb6  UR                  UUU5      nUR                  u  nnnUR                  S
US
5      nUR                  b1  UR                  UR                  SS	5      5      R                  SS	5      nUR                  U5      nUc  UnO"UR                  (       a  UR                  U5      nUR                  U5      nUR!                  U5      nUR#                  U5      R%                  5       nUR#                  U5      R%                  5       nUR#                  U5      R%                  5       n[&        R(                  R+                  UUUX@R,                  S9nUR/                  UR0                  5      nUR3                  U5      nW	(       Ga  UGb  [        U[4        5      (       d  [7        UR9                  S5      5      n[;        U5      [;        U R<                  5      s=:X  a  [;        U	5      :X  d<  O  [?        S[;        U5       S[;        U R<                  5       S[;        U	5       S35      e[A        [C        XpR<                  U	5      5       H  u  nu  nnnUc  M  [        U[D        RF                  5      (       a  UR                  S:w  a  [?        S5      eUR                  S   UR                  S   :w  a.  [?        SUR                  S    SUR                  S    SU 35      e[        U[6        5      (       d  M  [;        U5      UR                  S   :X  a  M  [?        SUR                  S    S[;        U5       SU 35      e   OS /[;        U R<                  5      -  n[C        XR<                  U RH                  U RJ                  U5       GH  u  nnnnnSn[        U[6        5      (       a  [M        S U 5       5      (       a  SnOUS:X  a  SnU(       a  ML  UGb  UR/                  [D        RN                  5      n[        U[6        5      (       d  U/UR                  S   -  nUR                  S   n[Q        U5       GH3  nU" US S 2US S 2S S 24   5      n U" US S 2US S 2S S 24   5      n!UR#                  U 5      R%                  5       n UR#                  U!5      R%                  5       n![&        R(                  R+                  UU U!U R,                  S9n"U"R/                  UR0                  5      n"UR3                  U"5      n"[R        RT                  " US S 2US S 2S S 24   UU"R                  S   U"R                  S	   5      n#U#R/                  UR0                  URV                  S9n#X&U   U"U#-  -  -   nGM6     GM  U" U5      n U" U5      n!UR#                  U 5      R%                  5       n UR#                  U!5      R%                  5       n![&        R(                  R+                  UU U!U R,                  S9nUR/                  UR0                  5      nUR3                  U5      nX&U-  -   nGM     URX                  S   " U5      nURX                  S   " U5      nUS:X  a$  UR                  S
S5      R[                  UWWW5      nUR\                  (       a  X(-   nX!R^                  -  nU$ )Nr   r!  r  Fr"  r7   r   r  r	   rH   )r.  r/  r$  r%  r&  r'  r(  r)  r*  r+  r,  c              3   *   #    U  H	  oS :H  v   M     g7fr.  r  r/  s     rx   r  :IPAdapterXFormersAttnProcessor.__call__.<locals>.<genexpr>  s     15a65r3  T)r/  r  r  )0r   r  r   r   r   r^   r   r  r  r#  rf  rK   rd   rb   r&  re   rf   r   r0  r   r   r   r   r   r   r   r   r  r  r   rY   r[   r4  r5  r   rB  r   r  r  ry  r  r
   r6  r   rk   r   r.   r-   )$ru   r  r   r   r   r  rY   r   r  r   r  r7  r  r   r  r  r  r  r   rh  r   r  r  r8  r  r9  r:  r   r  r;  r<  r  r=  r>  r@  rA  s$                                       rx   r   'IPAdapterXFormersAttnProcessor.__call__U  sk    ! !,/77:O7%'7C $ =wH[kpq/55a84??1;MM)!XgXq.9*1gh>:; (8
 ( --mBM"''
?1>1D1D.J)..zFUNS]]^_abcM $9#@MF[FaFa 	'
OQ %!88ZdeN "/!4!4A|Q+222|RHN??& OOM,C,CAq,IJTTUVXYZM		-( ($1!__$($C$CDY$Z!ii-.		/0&&u-88:$$S)446&&u-88: ??3<M<M @ 
 &((5..}=+!"2D99'+,<,F,Fq,I'J$,-TZZYCHXDYY$<SAQ=R<S T77:4::6G H 012!5  ;D,jj:JK;66eX  <$)$==a",!_# 
  ::a=HNN1,==","3DJJqM? C88@q8I7J+V[U\!^#  &eT223u:TU;V","3DJJqM? C558ZLE7!T# #;, %)6C

O#;  LO **dllDLLJZLG(%'4 eT**15111#aZDt'#wwu}}5)%66%*Gdjjm$;E-1ZZ]*!&'9!:A%,-EaAqj-Q%RF'./G1aQR
/S'TH%)%;%;F%C%N%N%PF'+'='=h'G'R'R'TH8@8_8_ %vxD<M<M 9` 95 9R8T8TUZU`U`8a58<8N8NOh8i5.D.O.O $Q1aZ 0 * 9 ? ? B 9 ? ? B	/O />.@.@u{{[`[g[g.@.hO,9!HHadsHs<t,tM+ ";. "))A!B#*+C#D!%!7!7!?!J!J!L#'#9#9(#C#N#N#P3;<<3Z3Z!688I8I 4[ 40 4L3N3Nu{{3[0373I3IJb3c0(5@X8X(XgLl A}5A}5?)33B;CCJPWY_afgM##)4M%(B(BBrz   )r   r   r   r   rY   r   r  )NrE  r;   NrF  )r   r;  r<  r=  r>  r   r   rP   r   r   r  r  r   rD  rE  rF  s   @rx   r   r   "  s    , !+/
 x(
 
F >B6:,08<rr ((r  ((9(9:	r
 !!2!23r u(()r r #5#4#45r rrz   r   c                      ^  \ rS rSrSr  SS\S\S\S\S\4
U 4S jjjr    SS	\S
\	R                  S\	R                  S\\	R                     S\	R                  S\	R                  S\	R                  4S jjrSrU =r$ )!SD3IPAdapterJointAttnProcessor2_0i
  a"  
Attention processor for IP-Adapter used typically in processing the SD3-like self-attention projections, with
additional image-based information and timestep embeddings.

Args:
    hidden_size (`int`):
        The number of hidden channels.
    ip_hidden_states_dim (`int`):
        The image feature dimension.
    head_dim (`int`):
        The number of head channels.
    timesteps_emb_dim (`int`, defaults to 1280):
        The number of input channels for timestep embedding.
    scale (`float`, defaults to 0.5):
        IP-Adapter scale.
r   ip_hidden_states_dimr  timesteps_emb_dimrY   c                   > [         TU ]  5         SSKJnJn  U" XBS-  SSS9U l        [        R                  " X!SS9U l        [        R                  " X!SS9U l	        U" US5      U l
        U" US5      U l        U" US5      U l        XPl        g )Nr7   )AdaLayerNormr:   r	   r  )
output_dimnorm_eps	chunk_dimFrM   )rO   rP   rQ   r[  r:   norm_ipr   rc   r   r  r_   r`   	norm_ip_krY   )	ru   r   rX  r  rY  rY   r[  r:   rw   s	           rx   rP   *SD3IPAdapterJointAttnProcessor2_0.__init__  s     	 	9#$5YZBZeiuvwyy!5Oyy!5Oh-h- 40
rz   r  r   r   r   r   r  r}   c                 	   UnUR                   S   nUR                  U5      n	UR                  U5      n
UR                  U5      nU
R                   S   nXR                  -  nU	R                  USUR                  U5      R                  SS5      n	U
R                  USUR                  U5      R                  SS5      n
UR                  USUR                  U5      R                  SS5      nU	nU
nUnUR                  b  UR                  U	5      n	UR                  b  UR                  U
5      n
UGb>  UR                  U5      nUR                  U5      nUR                  U5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nUR                  b  UR                  U5      nUR                  b  UR                  U5      n[        R                  " U	U/SS9n	[        R                  " U
U/SS9n
[        R                  " UU/SS9n[         R"                  " XUSSS	9nUR                  SS5      R%                  USUR                  U-  5      nUR'                  U	R(                  5      nUbQ  USS2SUR                   S   24   USS2UR                   S   S24   p2UR*                  (       d  UR-                  U5      nU R.                  S:w  Gae  UGba  U R1                  XVS
9nU R3                  U5      nU R5                  U5      nUR                  USUR                  U5      R                  SS5      nUR                  USUR                  U5      R                  SS5      nU R                  U5      n	U R                  U5      nU R7                  U5      n[        R                  " UU/SS9n
[        R                  " UU/SS9n[         R"                  " XUSSS	9nUR                  SS5      R                  USUR                  U-  5      nUR'                  U	R(                  5      nX%U R.                  -  -   nUR8                  S   " U5      nUR8                  S   " U5      nUb  X#4$ U$ )a  
Perform the attention computation, integrating image features (if provided) and timestep embeddings.

If `ip_hidden_states` is `None`, this is equivalent to using JointAttnProcessor2_0.

Args:
    attn (`Attention`):
        Attention instance.
    hidden_states (`torch.FloatTensor`):
        Input `hidden_states`.
    encoder_hidden_states (`torch.FloatTensor`, *optional*):
        The encoder hidden states.
    attention_mask (`torch.FloatTensor`, *optional*):
        Attention mask.
    ip_hidden_states (`torch.FloatTensor`, *optional*):
        Image embeddings.
    temb (`torch.FloatTensor`, *optional*):
        Timestep embeddings.

Returns:
    `torch.FloatTensor`: Output hidden states.
r   rH   r7   r	   Nr  r  Fr  )r  )r   rd   re   rf   r   r  r  r_   r`   ri   rg   rh   ro   rp   r   r  rr   rN   r   r   r   rW   rn   rY   r_  r   r  r`  rk   )ru   r  r   r   r   r   r  r  r   r   r  r  rR   r  	img_queryimg_key	img_valuer  r  r  norm_ip_hidden_statesr=  r>  s                          rx   r   *SD3IPAdapterJointAttnProcessor2_0.__call__1  s   > !"((+
 		-(ii&		-(IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP		;;"KK&E;;"++c"C !,/3?T/U,-1__=R-S*/3?T/U,/O/T/TB

H0i1o - .L-P-PB

H.i1o + 0P/T/TB

H0i1o -   ,373D3DEe3f0  ,151B1BCa1b.IIu&FGQOE))S"@AqICIIu&FGQOE66u5TWchi%//15==j"djj[cNcd%((5 , a!48>>!#4!445a!2!445 1 (((,8M(N% ::?/;$(LL1AL$M! \\"78F||$9:H [[RXFPPQRTUVF}}ZTZZJTTUVXYZH KK	*Ekk'*G^^F+F ))Wf-15CIIy(3;E ==e%[^jop/99!Q?DDZQSUYU_U_bjUjk/225;;?)tzz,IIM A}5A}5 , 77  rz   )r_  r`  r`   r_   rY   r   r  )i   g      ?r:  )r   r;  r<  r=  r>  r?  r  rP   r   r   r  r   r   rD  rE  rF  s   @rx   rW  rW  
  s    , "& " 	
   2 486:.2"&A!A! ((A!  %00	A!
 !!2!23A!  ++A! A! 
		A! A!rz   rW  c                       \ rS rSrSrS r   SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\R                  4S jjrSrg)PAGIdentitySelfAttnProcessor2_0i  
Processor for implementing PAG using scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
PAG reference: https://huggingface.co/papers/2403.17377
c                 D    [        [        S5      (       d  [        S5      eg )NrN   z_PAGIdentitySelfAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   (PAGIdentitySelfAttnProcessor2_0.__init__  s%    q899q  :rz   Nr  r   r   r   r  r}   c           	      B   UnUR                   b  UR                  X%5      nUR                  nUS:X  a3  UR                  u  ppUR                  XX-  5      R	                  SS5      nUR                  S5      u  pUR                  u  pnUb<  UR                  XNU5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      n[        R                  " UUUUSSS9nUR	                  SS5      R                  USUR                  U-  5      nUR                  UR                   5      nUR"                  S   " U5      nUR"                  S   " U5      nUS:X  a$  UR	                  SS	5      R                  UW	W
W5      nUR                  u  pnUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUR                  UR                   5      nUR"                  S   " U5      nUR"                  S   " U5      nUS:X  a$  UR	                  SS	5      R                  UW	W
W5      n[$        R&                  " X/5      nUR(                  (       a  X&-   nX!R*                  -  nU$ )
Nr  r7   r	   rH   r  Fr  r   r  )r^   r   r   r  r  r  r#  r   rK   rd   re   rf   rr   rN   r   r   r   rk   r   r  r.   r-   )ru   r  r   r   r   r  r  r  r   r  r  r  r  r   r  r   r   r  r  rR   r  s                        rx   r   (PAGIdentitySelfAttnProcessor2_0.__call__  s|    !( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM 0=/B/B1/E, *;)@)@&
Q%!88ZdeN ,00ZZ^MaMabdMefN??& $0A0K0KAq0Q R \ \]^`a b		+,ii)*		+,IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP ::33RW
 .771=EEjRTVZV`V`ckVkl-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o *;)@)@&
Q??& $0A0K0KAq0Q R \ \]^`a b II&78-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o 		#4"HI##)4M%(B(BBrz   r  r  r   r;  r<  r=  r>  rP   r   r   r  r   rB  r   rD  r  rz   rx   ri  ri    s    
 >B6:,0TT ((T  ((9(9:	T
 !!2!23T u(()T 
T Trz   ri  c                       \ rS rSrSrS r   SS\S\R                  S\	\R                     S\	\R                     S	\	\R                     S
\R                  4S jjrSrg)"PAGCFGIdentitySelfAttnProcessor2_0i  rj  c                 D    [        [        S5      (       d  [        S5      eg )NrN   zbPAGCFGIdentitySelfAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.r  r  s    rx   rP   +PAGCFGIdentitySelfAttnProcessor2_0.__init__  s%    q899t  :rz   Nr  r   r   r   r  r}   c           	      v   UnUR                   b  UR                  X%5      nUR                  nUS:X  a3  UR                  u  ppUR                  XX-  5      R	                  SS5      nUR                  S5      u  pn[        R                  " X/5      nUR                  u  pnUb<  UR                  XOU5      nUR                  XR                  SUR                  S   5      nUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUR                  U5      nUR                  U5      nUR                  S   nUUR                  -  nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      nUR                  USUR                  U5      R	                  SS5      n[        R                  " UUUUSSS9nUR	                  SS5      R!                  USUR                  U-  5      nUR#                  UR$                  5      nUR&                  S	   " U5      nUR&                  S   " U5      nUS:X  a$  UR	                  SS
5      R!                  UW	W
W5      nUR                  u  pnUR                  b1  UR                  UR	                  SS5      5      R	                  SS5      nUR                  U5      nUnUR#                  UR$                  5      nUR&                  S	   " U5      nUR&                  S   " U5      nUS:X  a$  UR	                  SS
5      R!                  UW	W
W5      n[        R                  " X/5      nUR(                  (       a  X&-   nX!R*                  -  nU$ )Nr  r7   r	   r   rH   r  Fr  r   r  )r^   r   r   r  r  r  r   r  r#  r   rK   rd   re   rf   rr   rN   r   r   r   rk   r.   r-   )ru   r  r   r   r   r  r  r  r   r  r  r  r  r  r   r  r   r   r  r  rR   r  s                         rx   r   +PAGCFGIdentitySelfAttnProcessor2_0.__call__$  s    !( --mBM"''
?1>1D1D.J)..zFNS]]^_abcM FSEXEXYZE[B1B!II';&OP *;)@)@&
Q%!88ZdeN ,00ZZ^MaMabdMefN??& $0A0K0KAq0Q R \ \]^`a b		+,ii)*		+,IIbM	

*

:r4::x@JJ1aPhhz2tzz8<FFq!L

:r4::x@JJ1aP ::33RW
 .771=EEjRTVZV`V`ckVkl-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o *;)@)@&
Q??& $0A0K0KAq0Q R \ \]^`a b		+,!-00= !KKN+<= KKN+<=? 1 ; ;B C K KJX_agin o 		#4"HI##)4M%(B(BBrz   r  r  ro  r  rz   rx   rq  rq    s    
 >B6:,0XX ((X  ((9(9:	X
 !!2!23X u(()X 
X Xrz   rq  c                   V    \ rS rSrSrS\S\R                  S\R                  4S jrSr	g)	re  i  z<
Processor for implementing multiscale quadratic attention.
r  r   r}   c                    UR                   SS  u  p4X4-  UR                  :  a  SnOSnUn[        UR                  5       5      u  pxp4UR                  n	UR                  SS5      nUR                  U5      n
UR                  U5      nUR                  U5      n[        R                  " XU/SS9nUR                  SS5      nU/nUR                   H  nUR                  U" U5      5        M     [        R                  " USS9nU(       a  UR                  [        R                  S9nUR                  USSUR                  -  X4-  5      nUR!                  SS	S9u  pnUR#                  U
5      n
UR#                  U5      nU(       a"  UR%                  XU5      nUR                  U	S9nOUR'                  XU5      n[        R                  " X'SX445      nUR)                  UR                  SS5      5      R                  SS5      nUR*                  S
:X  a2  UR-                  UR                  SS5      5      R                  SS5      nOUR-                  U5      nUR.                  (       a  X&-   nU$ )Nr  TFr7   rH   r   r  rl  r	   rE   )r   rZ  r  r  r   movedimrd   re   rf   r   r  ra  rl   r   rn  r   r  rc  rq  rv  rk   r\  rd  r.   )ru   r  r   r  r  use_linear_attentionr  r   r   original_dtyper   r  r  multi_scale_qkvblocks                  rx   r   'SanaMultiscaleAttnProcessor2_0.__call__  sD   %++BC0>D333#' #(  '+M,>,>,@'A$
v&,,%--a4		-(ii&		-(		5u"51=%--b!4(/++E""5#78 , 		/q9),,5==,AM%--j"a$BYBY>Y[a[ij)//q/9E!!%($ 77EJM),,>,BM ::5uMMm"f5TUM$9$9!R$@AII"aP>>Z' MM-*?*?2*FGOOPRTUVM MM-8M##)4Mrz   r  N)
r   r;  r<  r=  r>  rX  r   rB  r   rD  r  rz   rx   re  re    s-    4: 45<< 4\a\h\h 4rz   re  c                       \ rS rSrSrS rSrg)LoRAAttnProcessori  z1
Processor for implementing attention with LoRA.
c                     g r9  r  r  s    rx   rP   LoRAAttnProcessor.__init__      rz   r  Nr   r;  r<  r=  r>  rP   rD  r  rz   rx   r  r        rz   r  c                       \ rS rSrSrS rSrg)LoRAAttnProcessor2_0i  zb
Processor for implementing attention with LoRA (enabled by default if you're using PyTorch 2.0).
c                     g r9  r  r  s    rx   rP   LoRAAttnProcessor2_0.__init__  r  rz   r  Nr  r  rz   rx   r  r    r  rz   r  c                       \ rS rSrSrS rSrg)LoRAXFormersAttnProcessori  z@
Processor for implementing attention with LoRA using xFormers.
c                     g r9  r  r  s    rx   rP   "LoRAXFormersAttnProcessor.__init__  r  rz   r  Nr  r  rz   rx   r  r    r  rz   r  c                       \ rS rSrSrS rSrg)LoRAAttnAddedKVProcessori  zr
Processor for implementing attention with LoRA with extra learnable key and value matrices for the text encoder.
c                     g r9  r  r  s    rx   rP   !LoRAAttnAddedKVProcessor.__init__  r  rz   r  Nr  r  rz   rx   r  r    r  rz   r  c                       \ rS rSrSr  SS\S\R                  S\\R                     S\\R                     S\R                  4
S	 jjr	S
r
g)SanaLinearAttnProcessor2_0i  A
Processor for implementing scaled dot-product linear attention.
Nr  r   r   r   r}   c                    UR                   nUc  UnUR                  U5      nUR                  U5      nUR                  U5      nUR                  b  UR	                  U5      nUR
                  b  UR                  U5      nUR                  SS5      R                  SUR                  S45      nUR                  SS5      R                  SUR                  S45      R                  SS5      nUR                  SS5      R                  SUR                  S45      n[        R                  " U5      n[        R                  " U5      nUR                  5       UR                  5       UR                  5       pn[        R                  " USSSS9n[        R                  " X5      n	[        R                  " X5      nUS S 2S S 2S S24   US S 2S S 2SS 24   S	-   -  nUR                  SS5      R                  SS5      nUR!                  U5      nUR"                  S
   " U5      nUR"                  S   " U5      nU[        R$                  :X  a  UR'                  SS5      nU$ )Nr7   r	   rH   r   rh  ri  r;   rj  r|  r       )r   rd   re   rf   r_   r`   r  r  r   rr   relur  r  r   rm  r  r   rk   ry  clip)
ru   r  r   r   r   rz  r   r  r  rp  s
             rx   r   #SanaLinearAttnProcessor2_0.__call__  s    ',, ($1!		-(ii-.		/0;;"KK&E;;"++c"C1%//DJJ3CDmmAq!++A

B/?@JJ1aP1%//DJJ3CDuffSk!KKM399;Ee\
#Fe)V3%aCRCi0M!Q)4Lu4TU%--a3==aC%((8A}5A}5U]]*)..vu=Mrz   r  r:  r  r  rz   rx   r  r    sc     9=15,, ||,  (5	,
 !., 
, ,rz   r  c                       \ rS rSrSr  SS\S\R                  S\\R                     S\\R                     S\R                  4
S	 jjr	S
r
g) PAGCFGSanaLinearAttnProcessor2_0i  r  Nr  r   r   r   r}   c                 R   UR                   nUR                  S5      u  pgn[        R                  " Xg/5      nUR	                  U5      n	UR                  U5      n
UR                  U5      nU	R                  SS5      R                  SUR                  S45      n	U
R                  SS5      R                  SUR                  S45      R                  SS5      n
UR                  SS5      R                  SUR                  S45      n[        R                  " U	5      n	[        R                  " U
5      n
U	R                  5       U
R                  5       UR                  5       pn	[        R                  " USSSS9n[        R                  " X5      n[        R                  " X5      nUS S 2S S 2S S24   US S 2S S 2SS 24   S	-   -  nUR                  SS5      R                  SS5      nUR!                  U5      nUR"                  S
   " U5      nUR"                  S   " U5      nUR                  U5      R!                  U5      nUR"                  S
   " U5      nUR"                  S   " U5      n[        R                  " Xx/5      nU[        R$                  :X  a  UR'                  SS5      nU$ )Nr   r7   r	   rH   rh  ri  r;   rj  r|  r   r  r  )r   r  r   r  rd   re   rf   r  r  r   rr   r  r  r  rm  r  r   rk   ry  r  )ru   r  r   r   r   rz  r  r  r   r   r  r  rp  s                rx   r   )PAGCFGSanaLinearAttnProcessor2_0.__call__  sS    ',,EREXEXYZE[B1B!II';&OP		+,ii)*		+,1%//DJJ3CDmmAq!++A

B/?@JJ1aP1%//DJJ3CDuffSk!KKM399;Ee\
#Fe)!LL7-aCRCi8<MaQRTVTWi<X[`<`a-55a;EEaK-00@ KKN+<= KKN+<= !II&78;;NK KKN+<= KKN+<=		#4"HIU]]*)..vu=Mrz   r  r:  r  r  rz   rx   r  r    sc     9=15// ||/  (5	/
 !./ 
/ /rz   r  c                       \ rS rSrSr  SS\S\R                  S\\R                     S\\R                     S\R                  4
S	 jjr	S
r
g)%PAGIdentitySanaLinearAttnProcessor2_0iJ  r  Nr  r   r   r   r}   c                    UR                   nUR                  S5      u  pgUR                  U5      nUR                  U5      n	UR	                  U5      n
UR                  SS5      R                  SUR                  S45      nU	R                  SS5      R                  SUR                  S45      R                  SS5      n	U
R                  SS5      R                  SUR                  S45      n
[        R                  " U5      n[        R                  " U	5      n	UR                  5       U	R                  5       U
R                  5       pn[        R                  " U
SSSS9n
[        R                  " X5      n[        R                  " X5      nUR                   [        R                  [        R                  4;   a  UR                  5       nUS S 2S S 2S S24   US S 2S S 2SS 24   S	-   -  nUR!                  SS5      R                  SS5      nUR#                  U5      nUR$                  S
   " U5      nUR$                  S   " U5      nUR	                  U5      R#                  U5      nUR$                  S
   " U5      nUR$                  S   " U5      n[        R&                  " Xg/5      nU[        R                  :X  a  UR)                  SS5      nU$ )Nr	   r7   rH   r   rh  ri  r;   rj  r|  r   r  r  )r   r  rd   re   rf   r  r  r   rr   r  r  r  r   rm  ry  rz  r  r   rk   r  r  )ru   r  r   r   r   rz  r  r   r   r  r  rp  s               rx   r   .PAGIdentitySanaLinearAttnProcessor2_0.__call__O  sf    ',,/</B/B1/E,		+,ii)*		+,1%//DJJ3CDmmAq!++A

B/?@JJ1aP1%//DJJ3CDuffSk!KKM399;Ee\
#Fe)!LL7""u}}enn&EE 1 7 7 9-aCRCi8<MaQRTVTWi<X[`<`a-55a;EEaK-00@ KKN+<= KKN+<= !II&78;;NK KKN+<= KKN+<=		#4"HIU]]*)..vu=Mrz   r  r:  r  r  rz   rx   r  r  J  sc     9=1511 ||1  (5	1
 !.1 
1 1rz   r  c                       \ rS rSrS rSrg)FluxAttnProcessor2_0i  c                 <    Sn[        SSU5        SSKJn  U" U0 UD6$ )Nzq`FluxAttnProcessor2_0` is deprecated and this will be removed in a future version. Please use `FluxAttnProcessor`r  r  r7   FluxAttnProcessorr   transformers.transformer_fluxr  clsr  r  r  r  s        rx   __new__FluxAttnProcessor2_0.__new__  s,     R('3FGD $1&11rz   r  Nr   r;  r<  r=  r  rD  r  rz   rx   r  r        2rz   r  c                       \ rS rSrSrS rSrg)FluxSingleAttnProcessor2_0i  r  c                 <    Sn[        SSU5        SSKJn  U" U0 UD6$ )Nz|`FluxSingleAttnProcessor` is deprecated and will be removed in a future version. Please use `FluxAttnProcessorSDPA` instead.r  r  r7   r  r  r  s        rx   r  "FluxSingleAttnProcessor2_0.__new__  s,     ].9LMD $1&11rz   r  Nr   r;  r<  r=  r>  r  rD  r  rz   rx   r  r    s    2rz   r  c                       \ rS rSrS rSrg)FusedFluxAttnProcessor2_0i  c                 <    Sn[        SSU5        SSKJn  U" U0 UD6$ )Nzv`FusedFluxAttnProcessor2_0` is deprecated and this will be removed in a future version. Please use `FluxAttnProcessor`r  r  r7   r  r  r  s        rx   r  !FusedFluxAttnProcessor2_0.__new__  s,     W-w8KLD $1&11rz   r  Nr  r  rz   rx   r  r    r  rz   r  c                       \ rS rSrS rSrg)"FluxIPAdapterJointAttnProcessor2_0i  c                 <    Sn[        SSU5        SSKJn  U" U0 UD6$ )Nz`FluxIPAdapterJointAttnProcessor2_0` is deprecated and this will be removed in a future version. Please use `FluxIPAdapterAttnProcessor`r  r  r7   )FluxIPAdapterAttnProcessor)r   r  r  )r  r  r  r  r  s        rx   r  *FluxIPAdapterJointAttnProcessor2_0.__new__  s,     i6ATUM)4:6::rz   r  Nr  r  rz   rx   r  r    s    ;rz   r  c                       \ rS rSrS rSrg)FluxAttnProcessor2_0_NPUi  c                 J    Sn[        SSUSS9  SSKJn  U" 5       nSUl        U$ )	NzFluxAttnProcessor2_0_NPU is deprecated and will be removed in a future version. An alternative solution to use NPU Flash Attention will be provided in the future.r  r  Fr"  r7   r  _native_npur   r  r  _attention_backendr  r  r  r  r  r0   s         rx   r   FluxAttnProcessor2_0_NPU.__new__  s9    ^ 	 	,g7JZ_`D%'	'4	$rz   r  Nr  r  rz   rx   r  r        rz   r  c                       \ rS rSrS rSrg)FusedFluxAttnProcessor2_0_NPUi  c                 J    Sn[        SSUSS9  SSKJn  U" 5       nSUl        U$ )	NzFusedFluxAttnProcessor2_0_NPU is deprecated and will be removed in a future version. An alternative solution to use NPU Flash Attention will be provided in the future.r  r  Fr"  r7   r  
_fused_npur  )ru   r  r  r0   s       rx   r  %FusedFluxAttnProcessor2_0_NPU.__new__  s9    ^ 	 	17<O_deD%'	'3	$rz   r  Nr  r  rz   rx   r  r    r  rz   r  c                       \ rS rSrSrS rSrg)r   i  r  c                 <   Sn[        SSUSS9  [        SS5      (       a  [        S5      e[        5       (       a  [        SS	5      (       a  [        S
5      eSSKJn  [        U5      S:  d  UR                  SS 5      b  Sn[        SSU5        U" U0 UD6nSUl        U$ )NzXLAFluxFlashAttnProcessor2_0 is deprecated and will be removed in diffusers 1.0.0. An alternative solution to using XLA Flash Attention will be provided in the future.r   r  Fr"  r   r   r  r   r  r7   r  r   r|   zpartition_spec was not used in the processor implementation when it was added. Passing it is a no-op and support for it will be removed._native_xla)	r   r   r  r   r  r  r   r  r  r  s         rx   r  $XLAFluxFlashAttnProcessor2_0.__new__  s    ` 	 	0';N^cdU++VWW99-c599deeDt9q=FJJ'7>JA   &1DE%t6v6	'4	$rz   r  Nr  r  rz   rx   r   r     s    rz   r   )cr   r}  typingr   r   r   r   r   r   torch.nn.functionalr   
functionalrr   image_processorr
   utilsr   r   r   utils.import_utilsr   r   r   utils.torch_utilsr   r   
get_loggerr   r   r{  r   xformers.ops$torch_xla.experimental.custom_kernelr   torch_xla.runtimer   XLA_AVAILABLEr   r   rH  rX  r~  r  r1   r   r   r   r   r  r  r  r   r3  r<  rF  rL  rW  r   r   r   rs   r   r  r  r  r  r  r  r  r  r   r   r   r   r]   r   r   r   rW  ri  rq  re  r  r  r  r  r  r  r  r  r  r  r  r  r  r   ADDED_KV_ATTENTION_PROCESSORSCROSS_ATTENTION_PROCESSORSr   r  rz   rx   <module>r     s     9 9     4 > > d d F 
		H	%HC''H-MM Q&		 Q& Q&h		 4?3BII ?3D?
RYY ?
Df4 f4RE EPe299 eP@ @FI IXS! S!lY4 Y4xb4 b4JL4 L4^R! R!j[ [|Z! Z!z^! ^!BB4 B4JD4 D4ND DN[ [|o od[ [|q qhR RjB BJ_ _Dd dNx xvy yx] ]@g gTq299 qhobii odT TnY Yx")) :ERYY EPe ePeUXX__ ePh! h!V` `Fd dN9 9x    1 1h4 4n6 6r2 22 22 2; ;   @  	!  &  2M 2 22 2 	2
 2  2 2 2 2 "2 2 2 2 "2 2  #!2" !#2$ %2& '2( )2* +2, -2. /20  122 324 !526 728 "92: ;2< =2> )?2@ $A2B C2D E2F G2H %I2J *K2L "M2N #O2P &Q2R S2T U2V #W2X &Y2Z $[2\ ']2^ _2` a2b c2d e24 rz   