
    +hL                     P   S SK r S SKJrJrJrJrJrJrJr  S SK	r
S SKrS SKJrJrJrJrJrJr  SSKJrJr  SSKJrJrJrJr  SSKJr  SSKJrJr  SS	K J!r!  SS
K"J#r#  SSK$J%r%J&r&J'r'J(r(J)r)J*r*  SSK+J,r,  SSK-J.r.  SSK/J0r0  \&" 5       (       a  S SK1J2s  J3r4  Sr5OSr5\'Rl                  " \75      r8Sr9    S$S\:S\:S\;S\;4S jjr< S%S\Rz                  S\\R|                     S\?4S jjr@    S&S\\:   S\\\?\R                  4      S\\\:      S \\\;      4S! jjrB " S" S#\.\\\5      rCg)'    N)AnyCallableDictListOptionalTupleUnion)CLIPImageProcessorCLIPTextModelCLIPTokenizerCLIPVisionModelWithProjectionT5EncoderModelT5TokenizerFast   )PipelineImageInputVaeImageProcessor)FluxIPAdapterMixinFluxLoraLoaderMixinFromSingleFileMixinTextualInversionLoaderMixin)AutoencoderKL)FluxControlNetModelFluxMultiControlNetModel)FluxTransformer2DModel)FlowMatchEulerDiscreteScheduler)USE_PEFT_BACKENDis_torch_xla_availableloggingreplace_example_docstringscale_lora_layersunscale_lora_layers)randn_tensor   )DiffusionPipeline   )FluxPipelineOutputTFa  
    Examples:
        ```py
        >>> import torch
        >>> from diffusers.utils import load_image
        >>> from diffusers import FluxControlNetPipeline
        >>> from diffusers import FluxControlNetModel

        >>> base_model = "black-forest-labs/FLUX.1-dev"
        >>> controlnet_model = "InstantX/FLUX.1-dev-controlnet-canny"
        >>> controlnet = FluxControlNetModel.from_pretrained(controlnet_model, torch_dtype=torch.bfloat16)
        >>> pipe = FluxControlNetPipeline.from_pretrained(
        ...     base_model, controlnet=controlnet, torch_dtype=torch.bfloat16
        ... )
        >>> pipe.to("cuda")
        >>> control_image = load_image("https://huggingface.co/InstantX/SD3-Controlnet-Canny/resolve/main/canny.jpg")
        >>> prompt = "A girl in city, 25 years old, cool, futuristic"
        >>> image = pipe(
        ...     prompt,
        ...     control_image=control_image,
        ...     control_guidance_start=0.2,
        ...     control_guidance_end=0.8,
        ...     controlnet_conditioning_scale=1.0,
        ...     num_inference_steps=28,
        ...     guidance_scale=3.5,
        ... ).images[0]
        >>> image.save("flux.png")
        ```
base_seq_lenmax_seq_len
base_shift	max_shiftc                 4    XC-
  X!-
  -  nX5U-  -
  nX-  U-   nU$ N )image_seq_lenr'   r(   r)   r*   mbmus           k/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/pipelines/flux/pipeline_flux_controlnet.pycalculate_shiftr3   Z   s3     
	K$>?A%%A		Q	BI    encoder_output	generatorsample_modec                    [        U S5      (       a!  US:X  a  U R                  R                  U5      $ [        U S5      (       a   US:X  a  U R                  R                  5       $ [        U S5      (       a  U R                  $ [        S5      e)Nlatent_distsampleargmaxlatentsz3Could not access latents of provided encoder_output)hasattrr9   r:   moder<   AttributeError)r5   r6   r7   s      r2   retrieve_latentsr@   h   s}     ~}--+2I))00;;		/	/K84K))..00		+	+%%%RSSr4   num_inference_stepsdevice	timestepssigmasc                    Ub  Ub  [        S5      eUb  S[        [        R                  " U R                  5      R
                  R                  5       5      ;   nU(       d  [        SU R                   S35      eU R                  " S
X2S.UD6  U R                  n[        U5      nX14$ Ub  S[        [        R                  " U R                  5      R
                  R                  5       5      ;   nU(       d  [        SU R                   S35      eU R                  " S
XBS.UD6  U R                  n[        U5      nX14$ U R                  " U4S	U0UD6  U R                  nX14$ )a  
Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.

Args:
    scheduler (`SchedulerMixin`):
        The scheduler to get timesteps from.
    num_inference_steps (`int`):
        The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
        must be `None`.
    device (`str` or `torch.device`, *optional*):
        The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
    timesteps (`List[int]`, *optional*):
        Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
        `num_inference_steps` and `sigmas` must be `None`.
    sigmas (`List[float]`, *optional*):
        Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
        `num_inference_steps` and `timesteps` must be `None`.

Returns:
    `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
    second element is the number of inference steps.
zYOnly one of `timesteps` or `sigmas` can be passed. Please choose one to set custom valuesrC   zThe current scheduler class zx's `set_timesteps` does not support custom timestep schedules. Please check whether you are using the correct scheduler.)rC   rB   rD   zv's `set_timesteps` does not support custom sigmas schedules. Please check whether you are using the correct scheduler.)rD   rB   rB   r-   )

ValueErrorsetinspect	signatureset_timesteps
parameterskeys	__class__rC   len)	schedulerrA   rB   rC   rD   kwargsaccepts_timestepsaccept_sigmass           r2   retrieve_timestepsrS   v   s}   > !3tuu'3w/@/@AXAX/Y/d/d/i/i/k+ll .y/B/B.C Da b  	M)MfM''	!)n )) 
	 C(9(9):Q:Q(R(](](b(b(d$ee.y/B/B.C D_ `  	GvGG''	!)n )) 	 3MFMfM''	))r4   c            C         ^  \ rS rSrSrSrSS/r/ SQr  SMS\S	\	S
\
S\S\S\S\S\\\\   \\   \4   S\S\4U 4S jjjr     SNS\\\\   4   S\S\S\\R6                     S\\R8                     4
S jjr  SOS\\\\   4   S\S\\R6                     4S jjr       SPS\\\\   4   S\\\\\   4      S\\R6                     S\S\\R>                     S\\R>                     S\S\\    4S jjr!S r"S  r#        SQS! jr$\%S" 5       r&\%S# 5       r'\%S$ 5       r( SRS% jr)  SSS& jr*\+S' 5       r,\+S( 5       r-\+S) 5       r.\+S* 5       r/\R`                  " 5       \1" \25      SSSSS+SSS,SS-S.S+SSS+SSSSSSSSSSSS/S0SSS1/S4 S\\\\   4   S\\\\\   4      S2\\\\   4   S3\\\\\   4      S4\ S5\\   S6\\   S7\S8\\\       S9\ S:\\ \\    4   S;\\ \\    4   S<\3S=\\\\\   4      S>\\ \\    4   S\\   S?\\\Rh                  \\Rh                     4      S1\\R>                     S\\R>                     S\\R>                     S@\\3   SA\\\Rj                        SB\\3   SC\\\Rj                        SD\\R>                     SE\\R>                     SF\\   SG\6SH\\7\\84      SI\\9\\\7/S4      SJ\\   S\4@SK jj5       5       r:SLr;U =r<$ )TFluxControlNetPipeline   a{  
The Flux pipeline for text-to-image generation.

Reference: https://blackforestlabs.ai/announcing-black-forest-labs/

Args:
    transformer ([`FluxTransformer2DModel`]):
        Conditional Transformer (MMDiT) architecture to denoise the encoded image latents.
    scheduler ([`FlowMatchEulerDiscreteScheduler`]):
        A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
    vae ([`AutoencoderKL`]):
        Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
    text_encoder ([`CLIPTextModel`]):
        [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
        the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
    text_encoder_2 ([`T5EncoderModel`]):
        [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
        the [google/t5-v1_1-xxl](https://huggingface.co/google/t5-v1_1-xxl) variant.
    tokenizer (`CLIPTokenizer`):
        Tokenizer of class
        [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer).
    tokenizer_2 (`T5TokenizerFast`):
        Second Tokenizer of class
        [T5TokenizerFast](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5TokenizerFast).
z=text_encoder->text_encoder_2->image_encoder->transformer->vaeimage_encoderfeature_extractor)r<   prompt_embedscontrol_imageNrO   vaetext_encoder	tokenizertext_encoder_2tokenizer_2transformer
controlnetc                   > [         TU ]  5         [        U[        [        45      (       a  [        U5      nU R                  UUUUUUUUU	U
S9
  [        U SS 5      (       a/  S[        U R                  R                  R                  5      S-
  -  OSU l        [        U R                  S-  S9U l        [        U S5      (       a#  U R                   b  U R                   R"                  OSU l        S	U l        g )
N)
r[   r\   r^   r]   r_   r`   rO   ra   rW   rX   r[   r#   r%      )vae_scale_factorr]   M      )super__init__
isinstancelisttupler   register_modulesgetattrrN   r[   configblock_out_channelsrd   r   image_processorr=   r]   model_max_lengthtokenizer_max_lengthdefault_sample_size)selfrO   r[   r\   r]   r^   r_   r`   ra   rW   rX   rM   s              r2   rh   FluxControlNetPipeline.__init__   s     	j4-001*=J%)##!'/ 	 	
 W^^bdikoVpVpc$((//*L*L&MPQ&Q Rvw  1$BWBWZ[B[\/6t[/I/IdnnNhDNN++np 	! $' r4   r%      promptnum_images_per_promptmax_sequence_lengthrB   dtypec           
         U=(       d    U R                   nU=(       d    U R                  R                  n[        U[        5      (       a  U/OUn[        U5      n[        U [        5      (       a  U R                  XR                  5      nU R                  USUSSSSS9nUR                  nU R                  USSS9R                  n	U	R                  S   UR                  S   :  ag  [        R                  " X5      (       dL  U R                  R                  U	S S 2U R                  S	-
  S24   5      n
[         R#                  S
U SU
 35        U R%                  UR'                  U5      SS9S   nU R$                  R                  nUR'                  XTS9nUR                  u  pnUR)                  S	US	5      nUR+                  Xb-  US5      nU$ )N
max_lengthTFpt)paddingr|   
truncationreturn_lengthreturn_overflowing_tokensreturn_tensorslongestr~   r   r%   zXThe following part of your input was truncated because `max_sequence_length` is set to  	 tokens: output_hidden_statesr   rz   rB   )_execution_devicer\   rz   ri   strrN   r   maybe_convert_promptr]   r_   	input_idsshapetorchequalbatch_decoderr   loggerwarningr^   torepeatview)rt   rw   rx   ry   rB   rz   
batch_sizetext_inputstext_input_idsuntruncated_idsremoved_textrY   _seq_lens                 r2   _get_t5_prompt_embeds,FluxControlNetPipeline._get_t5_prompt_embeds   s    14110**00'44&&[
d788..v~~FF&& *&+ ' 
 %..**69UY*Zdd  $(<(<R(@@UcIuIu++88DLeLehiLilnLnIn9opLNN'(	,A
 ++N,=,=f,E\a+bcde##))%((u(D%++A &,,Q0EqI%**:+MwXZ[r4   c           
      r   U=(       d    U R                   n[        U[        5      (       a  U/OUn[        U5      n[        U [        5      (       a  U R                  XR                  5      nU R                  USU R                  SSSSS9nUR                  nU R                  USSS9R                  nUR                  S   UR                  S   :  aq  [        R                  " Xg5      (       dV  U R                  R                  US S 2U R                  S	-
  S24   5      n[        R                  S
U R                   SU 35        U R                  UR!                  U5      SS9n	U	R"                  n	U	R!                  U R                  R$                  US9n	U	R'                  S	U5      n	U	R)                  XB-  S5      n	U	$ )Nr|   TFr}   )r~   r|   r   r   r   r   r   r   r   r%   z\The following part of your input was truncated because CLIP can only handle sequences up to r   r   r   )r   ri   r   rN   r   r   r]   rr   r   r   r   r   r   r   r   r\   r   pooler_outputrz   r   r   )
rt   rw   rx   rB   r   r   r   r   r   rY   s
             r2   _get_clip_prompt_embeds.FluxControlNetPipeline._get_clip_prompt_embeds)  s    1411'44&&[
d788..v~~FFnn 00&+ % 
 %....SW.Xbb  $(<(<R(@@UcIuIu>>66q$JcJcfgJgjlJlGl7mnLNN--.i~G )).*;*;F*CZ_)` &33%((t/@/@/F/Fv(V &,,Q0EF%**:+MrRr4   prompt_2rY   pooled_prompt_embeds
lora_scalec	                    U=(       d    U R                   nUbw  [        U [        5      (       ab  Xl        U R                  b!  [
        (       a  [        U R                  U5        U R                  b!  [
        (       a  [        U R                  U5        [        U[        5      (       a  U/OUnUcH  U=(       d    Un[        U[        5      (       a  U/OUnU R                  UUUS9nU R                  UUUUS9nU R                  b6  [        U [        5      (       a!  [
        (       a  [        U R                  U5        U R                  b6  [        U [        5      (       a!  [
        (       a  [        U R                  U5        U R                  b  U R                  R                  OU R                  R                  n	[        R                  " UR                   S   S5      R#                  X9S9n
XVU
4$ )a.  

Args:
    prompt (`str` or `List[str]`, *optional*):
        prompt to be encoded
    prompt_2 (`str` or `List[str]`, *optional*):
        The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
        used in all text-encoders
    device: (`torch.device`):
        torch device
    num_images_per_prompt (`int`):
        number of images that should be generated per prompt
    prompt_embeds (`torch.FloatTensor`, *optional*):
        Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
        provided, text embeddings will be generated from `prompt` input argument.
    pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
        Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
        If not provided, pooled text embeddings will be generated from `prompt` input argument.
    clip_skip (`int`, *optional*):
        Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
        the output of the pre-final layer will be used for computing the prompt embeddings.
    lora_scale (`float`, *optional*):
        A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
)rw   rB   rx   )rw   rx   ry   rB   r%   r   rB   rz   )r   ri   r   _lora_scaler\   r   r    r^   r   r   r   r!   rz   r`   r   zerosr   r   )rt   rw   r   rB   rx   rY   r   ry   r   rz   text_idss              r2   encode_prompt$FluxControlNetPipeline.encode_promptU  s   F 1411 !j7J&K&K)   ,1A1A!$"3"3Z@"".3C3C!$"5"5zB'44&& )6H%/#%>%>zHH $(#?#?&; $@ $ 
 !66&;$7	 7 M ($ 3449I9I#D$5$5zB*$ 3449I9I#D$7$7D+/+<+<+H!!''dN^N^NdNd;;}2215q9<<F<XH<<r4   c                 F   [        U R                  R                  5       5      R                  n[	        U[
        R                  5      (       d  U R                  USS9R                  nUR                  X$S9nU R                  U5      R                  nUR                  USS9nU$ )Nr}   )r   r   r   dim)nextrW   rK   rz   ri   r   TensorrX   pixel_valuesr   image_embedsrepeat_interleave)rt   imagerB   rx   rz   r   s         r2   encode_image#FluxControlNetPipeline.encode_image  s    T''2245;;%..**5*FSSE4))%0==#556KQR5Sr4   c                 ,   / nUc  [        U[        5      (       d  U/n[        U5      U R                  R                  R
                  :w  a9  [        S[        U5       SU R                  R                  R
                   S35      eU H-  nU R                  XcS5      nUR                  US S S 24   5        M/     O[        U[        5      (       d  U/n[        U5      U R                  R                  R
                  :w  a9  [        S[        U5       SU R                  R                  R
                   S35      eU H  nUR                  U5        M     / nU H<  n[        R                  " U/U-  SS9nUR                  US	9nUR                  U5        M>     U$ )
NzK`ip_adapter_image` must have same length as the number of IP Adapters. Got z images and z IP Adapters.r%   zR`ip_adapter_image_embeds` must have same length as the number of IP Adapters. Got z image embeds and r   r   rB   )ri   rj   rN   r`   encoder_hid_projnum_ip_adaptersrF   r   appendr   catr   )rt   ip_adapter_imageip_adapter_image_embedsrB   rx   r   single_ip_adapter_imagesingle_image_embedss           r2   prepare_ip_adapter_image_embeds6FluxControlNetPipeline.prepare_ip_adapter_image_embeds  s    "*.55$4#5 #$(8(8(I(I(Y(YY abefvbwax  yE  FJ  FV  FV  Fg  Fg  Fw  Fw  Ex  xE  F  ,<'&*&7&78OYZ&[###$7a$@A ,< 5t<<+B*C'*+t/?/?/P/P/`/`` hil  nE  jF  iG  GY  Z^  Zj  Zj  Z{  Z{  ZK  ZK  YL  LY  Z  (?###$78 (? #%#/"'))-@,ADY,Y_`"a"5"8"8"8"G#**+>? $0
 '&r4   c           
      f  ^  UT R                   S-  -  S:w  d  UT R                   S-  -  S:w  a,  [        R                  ST R                   S-   SU SU S35        UbW  [        U 4S jU 5       5      (       d=  [	        ST R
                   S	U Vs/ s H  oT R
                  ;  d  M  UPM     sn 35      eUb  Ub  [	        S
U SU S35      eUb  Ub  [	        SU SU S35      eUc  Uc  [	        S5      eUbA  [        U[        5      (       d,  [        U[        5      (       d  [	        S[        U5       35      eUbA  [        U[        5      (       d,  [        U[        5      (       d  [	        S[        U5       35      eUb  Ub  [	        SU SU S35      eUb  Ub  [	        SU SU S35      eUbC  Ub@  UR                  UR                  :w  a&  [	        SUR                   SUR                   S35      eUb  U	c  [	        S5      eUb  U
c  [	        S5      eUb  US:  a  [	        SU 35      eg g s  snf )Nr#   r   z-`height` and `width` have to be divisible by z	 but are z and z(. Dimensions will be resized accordinglyc              3   @   >#    U  H  oTR                   ;   v   M     g 7fr,   )_callback_tensor_inputs).0krt   s     r2   	<genexpr>6FluxControlNetPipeline.check_inputs.<locals>.<genexpr>  s      F
7Y!---7Ys   z2`callback_on_step_end_tensor_inputs` has to be in z, but found zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.z Cannot forward both `prompt_2`: zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z2`prompt` has to be of type `str` or `list` but is z4`prompt_2` has to be of type `str` or `list` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: z)Cannot forward both `negative_prompt_2`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` .zIf `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`.zIf `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`.rv   z8`max_sequence_length` cannot be greater than 512 but is )rd   r   r   allrF   r   ri   r   rj   typer   )rt   rw   r   heightwidthnegative_promptnegative_prompt_2rY   negative_prompt_embedsr   negative_pooled_prompt_embeds"callback_on_step_end_tensor_inputsry   r   s   `             r2   check_inputs#FluxControlNetPipeline.check_inputs  sJ    T**Q./14AVAVYZAZ8[_`8`NN?@U@UXY@Y?ZZcdjckkpqvpw  x`  a .9# F
7YF
 C
 C
 DTEaEaDbbn  |^  pH  |^vw  ko  kG  kG  bGpq  |^  pH  oI  J  -";08N}o ^0 0  !m&?28*<RS`Ra b0 0  ^ 5w  FC)@)@TZ\`IaIaQRVW]R^Q_`aa!:h+D+DZX`bfMgMgSTXYaTbScdee&+A+M9/9J K*++]_  */E/Q;<M;N O*++]_ 
 $)?)K""&<&B&BB --:-@-@,A B.445Q8  $)=)E U  "-2O2W y  */BS/HWXkWlmnn 0I*e pHs   H.%H.c                 ,   [         R                  " XS5      nUS   [         R                  " U5      S S 2S 4   -   US'   US   [         R                  " U5      S S S 24   -   US'   UR                  u  pgnUR	                  Xg-  U5      nUR                  X4S9$ )Nr   ).r%   ).r#   r   )r   r   aranger   reshaper   )	r   r   r   rB   rz   latent_image_idslatent_image_id_heightlatent_image_id_widthlatent_image_id_channelss	            r2   _prepare_latent_image_ids0FluxControlNetPipeline._prepare_latent_image_ids%  s     !;;va8#3F#;ell6>RSTVZSZ>[#[ #3F#;ell5>QRVXYRY>Z#Z RbRhRhO7O+33":<T
  ""&">>r4   c                     U R                  XUS-  SUS-  S5      n U R                  SSSSSS5      n U R                  XS-  US-  -  US-  5      n U $ )Nr#   r      r%   r      )r   permuter   )r<   r   num_channels_latentsr   r   s        r2   _pack_latents$FluxControlNetPipeline._pack_latents4  sg     ,,z1aQVZ[Q[]^_//!Q1a3//*{uz.JL`cdLder4   c                    U R                   u  pEnS[        U5      US-  -  -  nS[        U5      US-  -  -  nU R                  XAS-  US-  US-  SS5      n U R                  SSSSSS5      n U R	                  XFS-  X5      n U $ )Nr#   r   r   r   r%   r   )r   intr   r   r   )r<   r   r   rd   r   num_patcheschannelss          r2   _unpack_latents&FluxControlNetPipeline._unpack_latents=  s     -4MM)
 c&k&6&:;<SZ$4q$89:,,zQ;
HPQMSTVWX//!Q1a3//*5.A6Qr4   c	                    S[        U5      U R                  S-  -  -  nS[        U5      U R                  S-  -  -  nXX44n	Ub*  U R                  XS-  US-  Xe5      n
UR                  XeS9U
4$ [	        U[
        5      (       a*  [        U5      U:w  a  [        S[        U5       SU S35      e[        XXeS9nU R                  XX#U5      nU R                  XS-  US-  Xe5      n
X4$ )Nr#   r   z/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.)r6   rB   rz   )
r   rd   r   r   ri   rj   rN   rF   r"   r   )rt   r   r   r   r   rz   rB   r6   r<   r   r   s              r2   prepare_latents&FluxControlNetPipeline.prepare_latentsO  s    c&kd&;&;a&?@ASZD$9$9A$=>?6A#==jTU+W\`aWaciq::V:9;KKKi&&3y>Z+GA#i.AQ R&<'gi 
 u&V$$W:NX]^99*PQkSX\]S]_em((r4   c
                 <   [        U[        R                  5      (       a  OU R                  R	                  XUS9nUR
                  S   n
U
S:X  a  UnOUnUR                  USS9nUR                  XgS9nU(       a!  U	(       d  [        R                  " U/S-  5      nU$ )N)r   r   r   r%   r   r   r#   )	ri   r   r   rp   
preprocessr   r   r   r   )rt   r   r   r   r   rx   rB   rz   do_classifier_free_guidance
guess_modeimage_batch_size	repeat_bys               r2   prepare_image$FluxControlNetPipeline.prepare_images  s     eU\\**((33EPU3VE ;;q>q "I .I''	q'94&zIIugk*Er4   c                     U R                   $ r,   )_guidance_scalert   s    r2   guidance_scale%FluxControlNetPipeline.guidance_scale  s    ###r4   c                     U R                   $ r,   )_joint_attention_kwargsr   s    r2   joint_attention_kwargs-FluxControlNetPipeline.joint_attention_kwargs  s    +++r4   c                     U R                   $ r,   )_num_timestepsr   s    r2   num_timesteps$FluxControlNetPipeline.num_timesteps  s    """r4   c                     U R                   $ r,   )
_interruptr   s    r2   	interrupt FluxControlNetPipeline.interrupt  s    r4         ?   g      @g        pilTr<   r   r   true_cfg_scaler   r   rA   rD   r   control_guidance_startcontrol_guidance_endrZ   control_modecontrolnet_conditioning_scaler6   r   r   negative_ip_adapter_image negative_ip_adapter_image_embedsr   r   output_typereturn_dictr   callback_on_step_endr   c!                    U=(       d    U R                   U R                  -  nU=(       d    U R                   U R                  -  n[        U[        5      (       d%  [        U[        5      (       a  [	        U5      U/-  nO[        U[        5      (       d%  [        U[        5      (       a  [	        U5      U/-  nOu[        U[        5      (       d`  [        U[        5      (       dK  [        U R
                  [        5      (       a  [	        U R
                  R                  5      OSn!U!U/-  U!U/-  pU R                  UUUUUUUUUUUU S9  Xl	        UU l
        SU l        Ub  [        U[        5      (       a  Sn"O3Ub!  [        U[        5      (       a  [	        U5      n"OUR                  S   n"U R                  n#U R                  R                   n$U R"                  b  U R"                  R%                  SS5      OSn%US:  =(       a    USLn&U R'                  UUUUU#UU U%S9u  nnn'U&(       a  U R'                  UUUUU#UU U%S9u  nnn(U R                  R(                  R*                  S-  n)[        U R
                  [,        5      (       Ga  U R/                  UUUU"U-  UU#U R0                  R                   S	9nUR                  S
S u  pgU R
                  R2                  c  SOSn*U R
                  R2                  c  [5        U R0                  R7                  U5      US9nXR0                  R(                  R8                  -
  U R0                  R(                  R:                  -  nUR                  SS u  n+n,U R=                  UU"U-  U)U+U,5      nUb  [        U[>        5      (       d  [A        S5      e[B        RD                  " U5      RG                  U#[B        RH                  S9nURK                  SS5      RM                  UR                  S   S5      nGO6[        U R
                  [        5      (       Ga  / n-U R
                  R                  S   R2                  c  SOSn*[O        U5       GH
  u  n.n/U R/                  U/UUU"U-  UU#U R0                  R                   S	9n/U/R                  S
S u  pgU R
                  R                  S   R2                  c  [5        U R0                  R7                  U/5      US9n/U/U R0                  R(                  R8                  -
  U R0                  R(                  R:                  -  n/U/R                  SS u  n+n,U R=                  U/U"U-  U)U+U,5      n/U-RQ                  U/5        GM     U-n[        U[        5      (       a#  [	        U5      [	        U5      :w  a  [A        S5      e[        U[        5      (       d  U/[	        U5      -  n/ n0U Hj  n1U1c  Sn1[B        RD                  " U15      RM                  U-S   R                  S   5      RG                  U#[B        RH                  S9nU0RQ                  U5        Ml     U0nU R                  R(                  R*                  S-  n)U RS                  U"U-  U)UUUR                   U#UU5      u  nn2U	c  [T        RV                  " SSU-  U5      OU	n	UR                  S   n3[Y        U3U RZ                  R(                  R%                  SS5      U RZ                  R(                  R%                  SS5      U RZ                  R(                  R%                  SS5      U RZ                  R(                  R%                  SS5      5      n4[]        U RZ                  UU#U	U4S9u  n5n[_        [	        U55      XRZ                  R`                  -  -
  S5      n6[	        U55      U l1        / n7[e        [	        U55      5       H  n.[g        X5       V8V9s/ s H>  u  n8n9S[i        U.[	        U55      -  U8:  =(       d    U.S-   [	        U55      -  U9:  5      -
  PM@     n:n8n9U7RQ                  [        U R
                  [,        5      (       a  U:S   OU:5        M     Uc  Ub,  Uc)  Uc&  [T        Rj                  " XvS4[T        Rl                  S9nO1Uc.  Uc+  Uc  Ub%  [T        Rj                  " XvS4[T        Rl                  S9nU R"                  c  0 U l
        Sn;Sn<Uc  Ub  U Ro                  UUU#U"U-  5      n;Uc  Ub  U Ro                  UUU#U"U-  5      n<U Rq                  US9 n=[O        U55       GH  u  n.n>U Rr                  (       a  M  U;b  U;U R                  S'   U>RM                  UR                  S   5      RG                  UR                   5      n?[        U R
                  [        5      (       a.  U R
                  R                  S   R(                  Rt                  n@O U R
                  R(                  Rt                  n@W@(       a  [B        RD                  " U
/U#S9OSnAUAb  WARM                  UR                  S   5      OSnA[        U7U.   [        5      (       a(  [g        UU7U.   5       VBV8s/ s H  u  nBn8UBU8-  PM     nCnBn8O$UnD[        UD[        5      (       a  WDS   nDWDU7U.   -  nCU R                  UUUWCU?S -  WAUUU'U2U R"                  SS!9u  nEnFU R                  R(                  Rt                  (       a  [B        RD                  " U
/U#S9OSnAUAb  WARM                  UR                  S   5      OSnAU R                  UU?S -  UAUUWEWFU'U2U R"                  SW*S"9S   nGU&(       aG  U<b  U<U R                  S'   U R                  UU?S -  WAUUWEWFU'U2U R"                  SU*S"9S   nHUHUWGUH-
  -  -   nGUR                   nIU RZ                  Rw                  WGU>USS#9S   nUR                   UI:w  a>  [B        Rx                  Rz                  R}                  5       (       a  URG                  WI5      nUb\  0 nJU H  nK[        5       UK   WJUK'   M     U" U U.U>WJ5      nLULR                  S$U5      nULR                  S%U5      nULR                  S&U5      nU.[	        U55      S-
  :X  d)  U.S-   U6:  a0  U.S-   U RZ                  R`                  -  S:X  a  U=R                  5         [        (       d  GM  [        R                  " 5         GM     SSS5        US':X  a  UnMOU R                  UXgU R                  5      nUU R0                  R(                  R:                  -  U R0                  R(                  R8                  -   nU R0                  R                  USS#9S   nMU R                  R                  UMUS(9nMU R                  5         U(       d  WM4$ [        WMS)9$ s  sn9n8f s  sn8nBf ! , (       d  f       N= f)*a  
Function invoked when calling the pipeline for generation.

Args:
    prompt (`str` or `List[str]`, *optional*):
        The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
        instead.
    prompt_2 (`str` or `List[str]`, *optional*):
        The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
        will be used instead
    height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
        The height in pixels of the generated image. This is set to 1024 by default for the best results.
    width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
        The width in pixels of the generated image. This is set to 1024 by default for the best results.
    num_inference_steps (`int`, *optional*, defaults to 50):
        The number of denoising steps. More denoising steps usually lead to a higher quality image at the
        expense of slower inference.
    sigmas (`List[float]`, *optional*):
        Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in
        their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed
        will be used.
    guidance_scale (`float`, *optional*, defaults to 7.0):
        Guidance scale as defined in [Classifier-Free Diffusion
        Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2.
        of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
        `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to
        the text `prompt`, usually at the expense of lower image quality.
    control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0):
        The percentage of total steps at which the ControlNet starts applying.
    control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0):
        The percentage of total steps at which the ControlNet stops applying.
    control_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,:
            `List[List[torch.Tensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`):
        The ControlNet input condition to provide guidance to the `unet` for generation. If the type is
        specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted
        as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or
        width are passed, `image` is resized accordingly. If multiple ControlNets are specified in `init`,
        images must be passed as a list such that each element of the list can be correctly batched for input
        to a single ControlNet.
    controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0):
        The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added
        to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set
        the corresponding scale as a list.
    control_mode (`int` or `List[int]`,, *optional*, defaults to None):
        The control mode when applying ControlNet-Union.
    num_images_per_prompt (`int`, *optional*, defaults to 1):
        The number of images to generate per prompt.
    generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
        One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
        to make generation deterministic.
    latents (`torch.FloatTensor`, *optional*):
        Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
        generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
        tensor will ge generated by sampling using the supplied random `generator`.
    prompt_embeds (`torch.FloatTensor`, *optional*):
        Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
        provided, text embeddings will be generated from `prompt` input argument.
    pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
        Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
        If not provided, pooled text embeddings will be generated from `prompt` input argument.
    ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
    ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):
        Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
        IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not
        provided, embeddings are computed from the `ip_adapter_image` input argument.
    negative_ip_adapter_image:
        (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
    negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):
        Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
        IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not
        provided, embeddings are computed from the `ip_adapter_image` input argument.
    output_type (`str`, *optional*, defaults to `"pil"`):
        The output format of the generate image. Choose between
        [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
    return_dict (`bool`, *optional*, defaults to `True`):
        Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple.
    joint_attention_kwargs (`dict`, *optional*):
        A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
        `self.processor` in
        [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
    callback_on_step_end (`Callable`, *optional*):
        A function that calls at the end of each denoising steps during the inference. The function is called
        with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
        callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
        `callback_on_step_end_tensor_inputs`.
    callback_on_step_end_tensor_inputs (`List`, *optional*):
        The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
        will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
        `._callback_tensor_inputs` attribute of your pipeline class.
    max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`.

Examples:

Returns:
    [`~pipelines.flux.FluxPipelineOutput`] or `tuple`: [`~pipelines.flux.FluxPipelineOutput`] if `return_dict`
    is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated
    images.
r%   )r   r   rY   r   r   r   r   ry   FNr   scale)rw   r   rY   r   rB   rx   ry   r   r   )r   r   r   r   rx   rB   rz   T)r6   r#   zB For `FluxControlNet`, `control_mode` should be an `int` or `None`)rz   r   zFor Multi-ControlNet, `control_mode` must be a list of the same  length as the number of controlnets (control images) specifiedr
  base_image_seq_len   max_image_seq_len   r)         ?r*   ffffff?)rD   r1   r   )totalr   r   i  )hidden_statescontrolnet_condcontrolnet_modeconditioning_scaletimestepguidancepooled_projectionsencoder_hidden_statestxt_idsimg_idsr   r  )r!  r%  r&  r'  r(  controlnet_block_samplescontrolnet_single_block_samplesr)  r*  r   r  controlnet_blocks_repeat)r  r<   rY   rZ   latent)r  )images)Krs   rd   ri   rj   rN   ra   r   netsr   r   r   r  r   r   r   r`   rz   r   getr   rn   in_channelsr   r   r[   input_hint_blockr@   encodeshift_factorscaling_factorr   r   rF   r   tensorr   longr   expand	enumerater   r   nplinspacer3   rO   rS   maxorderr  rangezipfloatr   uint8r   progress_barr  guidance_embedsstepbackendsmpsis_availablelocalspopupdateXLA_AVAILABLExm	mark_stepr   decoderp   postprocessmaybe_free_model_hooksr&   )Nrt   rw   r   r   r   r  r   r   rA   rD   r   r  r  rZ   r  r  rx   r6   r<   rY   r   r   r   r  r  r   r   r  r  r   r  r   ry   multr   rB   rz   r   do_true_cfgr   r   r   r-  height_control_imagewidth_control_imagecontrol_imagesicontrol_image_control_modescmoder   r.   r1   rC   num_warmup_stepscontrolnet_keepsekeepsr   negative_image_embedsrC  tr%  use_guidancer&  c
cond_scalecontrolnet_cond_scaler+  r,  
noise_predneg_noise_predlatents_dtypecallback_kwargsr   callback_outputsr   sN                                                                                 r2   __call__FluxControlNetPipeline.__call__  s   P K433d6K6KKI11D4I4II0$77JG[]a<b<b%()=%>BXAY%Y"0$77JG]_c<d<d#&'=#>BVAW#W 2D99*MacgBhBh0:4??Ld0e0e3t++,klD.//,-- %9 	+/'#9!5*G/Q 3 	 	
  .'=$ *VS"9"9JJvt$<$<VJ&,,Q/J''  && ?C>Y>Y>eD''++GT:ko 	 %q(H_D-H
 '!5"7 3!  	
		
  
 ""&*4%B&;$7% # 		&-  $//66BBaGdoo':;; ..#%(==&;hhnn / M *//4MF 150P0P0Xu^b$//7 01O[d e!.1M1M!MQUQYQYQ`Q`QoQo o =J<O<OPQPR<S9$&9 $ 2 2!!66(('! '!,44$%ijj$||L9<<V5::<V+00Q7>>}?R?RST?UWXY)ABBN040D0DQ0G0X0X0`ufj$%.}%=!>!%!3!3(!),AA*?!((.. "4 " !/ 4 4RS 9??''*;;C%5dhhoon6Uaj%kN&4txx7S7S&SW[W_W_WfWfWuWu%uN AO@T@TUVUW@X=(*=%)%7%7&"%::,,+&N %%n55 &>8 +M ,--#l2Cs=GY2Y X  lD11 ,~M0BBM%=E$||E299.:K:Q:QRS:TUXXY_glgqgqXr$$\2	 &
 )L  $//66BBaG$($8$8.. 	%
!! TZSaS!&9"9;NOgma(NN!!%%&:C@NN!!%%&94@NN!!%%lC8NN!!%%k48
 *<NN*
&	& s9~0CnnFZFZ0ZZ\]^!)n s9~&A   6MMDAq eAI.2Rq1uI6NQR6RSSM   ""z$//K^/_/_58ejk ' (,C,O%-2R2Z(*%1C288(T%&+B+J%15U5a!xx(:"((K&&.+-D( $'+B+N?? '22	L %04T4`$($H$H)022	%! %89\!),1>>+NZD001JK88GMM!$4588Gdoo/GHH#'??#7#7#:#A#A#Q#QL#'??#9#9#I#ILLX5<<(8H^b@H@T8??7==+;<Z^oa0$77478UWfghWi4j!k4jDAq!a%4jJ!kJ,I)!"7>>0Ea0H-!69K!KJ MQOO")$1$0'1%_%';*7$,+/+F+F % M\ MI(*I  FJEUEUE\E\ElElELL.!1&Arv  AI@T8??7==+;<Z^!--")%_%';*7-E4S$,+/+F+F %-E .  
 ,8Rg445NO%)%5%5&-!)D!)+H.D1I8W ( 0/3/J/J$)1I &6 & &N "0.JQ_D_2`!`J !(..--j!WRW-XYZ[==M1~~))6688")**]";'3&(O?-3Xa[* @';D!Q'X$.229gFG$4$8$8-$XM$4$8$8-$XM I**A9I/IqSTuX\XfXfXlXlNlpqNq '') =LLNQ - :V ("E **7F4CXCXYG!?!??488??C_C__GHHOOGO?BE((44U4TE 	##%8O!//n "l' :9s-   Av8Ew%v>
7Iww>w
w)	r   r  r   r   r  rs   rp   rr   rd   )NN)Nr%   rv   NN)r%   N)NNr%   NNrv   N)NNNNNNNNr,   )FF)=__name__
__module____qualname____firstlineno____doc__model_cpu_offload_seq_optional_componentsr   r   r   r   r   r   r   r   r	   r   r   r   r   r   r
   rh   r   r   r   r   rB   rz   r   r   FloatTensorrA  r   r   r   r   staticmethodr   r   r   r   r   propertyr   r   r  r  no_gradr   EXAMPLE_DOC_STRINGr   	Generatorr   boolr   r   r   rk  __static_attributes____classcell__)rM   s   @r2   rU   rU      s
   4 \+-@AK 8<04&'2&' &' $	&'
 !&' '&' %&' ,&' &9!:EBU<VXpp
&' 5&' .&' &'T )-%&#&)-'+/c49n%/  #/ !	/
 &/ $/h &')-	*c49n%*  #* &	*^ 59)-%&59<@#&&*P=c49n%P= 5d3i01P= &	P=
  #P=   1 12P= 'u'8'89P= !P= UOP=f	"'T #!&*+/ KoZ ? ?    2 !)Z %* D $ $ , , # #   ]]_12 )-4815=A # $##%(, #<?:=,08<CF/0MQ/359<@9=@DBFIM>BEI%* ;?KO9B#&Cs0c49n%s0 5d3i01s0 sDI~.	s0
 $E#tCy.$9:s0 s0 s0 }s0 !s0 e%s0 s0 !&eT%[&8 9s0 $E4;$67s0 *s0 uS$s)^45s0  (-UDK-?'@!s0"  (}#s0$ E%//43H"HIJ%s0& %++,'s0(   1 12)s0* 'u'8'89+s0, ##56-s0. "*$u||*<!=/s00 $,,>#?1s02 +343E*F3s04 !)):): ;5s06 (00A0A'B7s08 c]9s0: ;s0< !)c3h 8=s0> 'xc40@$0F'GH?s0@ -1IAs0B !Cs0 3 s0r4   rU   )r  r  r  r  )Nr:   )NNNN)DrH   typingr   r   r   r   r   r   r	   numpyr;  r   transformersr
   r   r   r   r   r   rp   r   r   loadersr   r   r   r   models.autoencodersr   "models.controlnets.controlnet_fluxr   r   models.transformersr   
schedulersr   utilsr   r   r   r   r    r!   utils.torch_utilsr"   pipeline_utilsr$   pipeline_outputr&   torch_xla.core.xla_modelcore	xla_modelrM  rL  
get_loggerrm  r   rx  r   rA  r3   r   ry  r   r@   rB   rS   rU   r-   r4   r2   <module>r     so    D D D    E p p 0 _ 9 9  . . / ))MM 
		H	% D 

 
 	

 
 ck
TLL
T-5eoo-F
T\_
T  *.15%)$(8*!#8* U3,-.8* S	"	8*
 T%[!8*vi0.0CEXZl i0r4   