+h |SSKrSSKJrJrJrJrJrJrJr SSK r SSK J s J r SSKJrJrJrJr SSKJr SSKJrJrJrJr SSKJrJrJrJrJrJ r SSK!J"r" SS K#J$r$ SS K%J&r&J'r'J(r(J)r)J*r*J+r+ SS K,J-r-J.r.J/r/J0r0J1r1 SS K2J3r3J4r4 SS K5J6r6 SSK7J8r8 SSK9J:r: SSK;JJ?r? \."5(a SSK@JAs JBrC SrDOSrD\/R"\F5rGSrHS"S\ RS\\ RS\K4SjjrLS#S\\MS\\\K\ R4S\\\MS\\\O4SjjrP"S S!\<\=\\\\8\:\5 rQg)$N)AnyCallableDictListOptionalTupleUnion)CLIPImageProcessor CLIPTextModel CLIPTokenizerCLIPVisionModelWithProjection)PipelineImageInput)FromSingleFileMixinIPAdapterMixinStableDiffusionLoraLoaderMixinTextualInversionLoaderMixin) AutoencoderKLControlNetModelImageProjectionMultiControlNetModelUNet2DConditionModelUNetMotionModel)adjust_lora_scale_text_encoder) MotionAdapter) DDIMSchedulerDPMSolverMultistepSchedulerEulerAncestralDiscreteSchedulerEulerDiscreteSchedulerLMSDiscreteScheduler PNDMScheduler)USE_PEFT_BACKENDis_torch_xla_availableloggingscale_lora_layersunscale_lora_layers)is_compiled_module randn_tensor)VideoProcessor) FreeInitMixin)AnimateDiffFreeNoiseMixin)DiffusionPipelineStableDiffusionMixin)AnimateDiffPipelineOutputTFak Examples: ```py >>> import torch >>> from PIL import Image >>> from tqdm.auto import tqdm >>> from diffusers import AnimateDiffVideoToVideoControlNetPipeline >>> from diffusers.utils import export_to_gif, load_video >>> from diffusers import AutoencoderKL, ControlNetModel, MotionAdapter, LCMScheduler >>> controlnet = ControlNetModel.from_pretrained( ... "lllyasviel/sd-controlnet-openpose", torch_dtype=torch.float16 ... ) >>> motion_adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM") >>> vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16) >>> pipe = AnimateDiffVideoToVideoControlNetPipeline.from_pretrained( ... "SG161222/Realistic_Vision_V5.1_noVAE", ... motion_adapter=motion_adapter, ... controlnet=controlnet, ... vae=vae, ... ).to(device="cuda", dtype=torch.float16) >>> pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear") >>> pipe.load_lora_weights( ... "wangfuyun/AnimateLCM", weight_name="AnimateLCM_sd15_t2v_lora.safetensors", adapter_name="lcm-lora" ... ) >>> pipe.set_adapters(["lcm-lora"], [0.8]) >>> video = load_video( ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/dance.gif" ... ) >>> video = [frame.convert("RGB") for frame in video] >>> from controlnet_aux.processor import OpenposeDetector >>> open_pose = OpenposeDetector.from_pretrained("lllyasviel/Annotators").to("cuda") >>> for frame in tqdm(video): ... conditioning_frames.append(open_pose(frame)) >>> prompt = "astronaut in space, dancing" >>> negative_prompt = "bad quality, worst quality, jpeg artifacts, ugly" >>> strength = 0.8 >>> with torch.inference_mode(): ... video = pipe( ... video=video, ... prompt=prompt, ... negative_prompt=negative_prompt, ... num_inference_steps=10, ... guidance_scale=2.0, ... controlnet_conditioning_scale=0.75, ... conditioning_frames=conditioning_frames, ... strength=strength, ... generator=torch.Generator().manual_seed(42), ... ).frames[0] >>> video = [frame.resize(conditioning_frames[0].size) for frame in video] >>> export_to_gif(video, f"animatediff_vid2vid_controlnet.gif", fps=8) ``` encoder_output generator sample_modec[US5(a!US:XaURRU5$[US5(a US:XaURR5$[US5(a UR$[ S5e)N latent_distsampleargmaxlatentsz3Could not access latents of provided encoder_output)hasattrr5r6moder8AttributeError)r1r2r3s څ/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.pyretrieve_latentsr=~s}~}--+2I))00;;  / /K84K))..00  + +%%%RSSnum_inference_stepsdevice timestepssigmasc UbUb [S5eUbS[[R"UR5R R 55;nU(d[SURS35eUR"S X2S.UD6 URn[U5nX14$UbS[[R"UR5R R 55;nU(d[SURS35eUR"S XBS.UD6 URn[U5nX14$UR"U4S U0UD6 URnX14$) a Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. Args: scheduler (`SchedulerMixin`): The scheduler to get timesteps from. num_inference_steps (`int`): The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` must be `None`. device (`str` or `torch.device`, *optional*): The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. timesteps (`List[int]`, *optional*): Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, `num_inference_steps` and `sigmas` must be `None`. sigmas (`List[float]`, *optional*): Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, `num_inference_steps` and `timesteps` must be `None`. Returns: `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the second element is the number of inference steps. zYOnly one of `timesteps` or `sigmas` can be passed. Please choose one to set custom valuesrAzThe current scheduler class zx's `set_timesteps` does not support custom timestep schedules. Please check whether you are using the correct scheduler.)rAr@rBzv's `set_timesteps` does not support custom sigmas schedules. Please check whether you are using the correct scheduler.)rBr@r@) ValueErrorsetinspect signature set_timesteps parameterskeys __class__rAlen) schedulerr?r@rArBkwargsaccepts_timesteps accept_sigmass r<retrieve_timestepsrRs}>!3tuu'3w/@/@AXAX/Y/d/d/i/i/k+ll .y/B/B.CDab  M)MfM'' !)n  ))   C(9(9):Q:Q(R(](](b(b(d$ee .y/B/B.CD_`  GvGG'' !)n  ))  3MFMfM''  ))r>c@^\rSrSrSrSr/SQr/SQrSTS\S\ S \ S \ \ \ 4S \S \ \\\\\\4S \ \\\\\\4S\S\4U4SjjjrSUS\\R<S\\R<S\\S\\ 4Sjjr!SVSjr"Sr#SWS\ S\R<4Sjjr$SWS\ 4Sjjr%Sr&SXS jr'S!r(SYS$\\R<S%\ S&\ S'\ S(\ S)\\ S*\\RRS+\\RTS,\\ \RV\\RV4S-\\R<S\ S.\,S\R<4S/jjr-SZS0jr.\/S15r0\/S25r1\/S35r2\/S45r3\/S55r4\/S65r5\Rl"5SSSSS7S#SSS8S9SS"SSSSSSSSS:S;SSS#SSSSS-/S4S$\\\7S<\\ \8\\84S%\\ S&\\ S=\ S>\,S?\\\ S@\\\SA\SB\SC\\ \8\\84SD\\ SE\S,\\ \RV\\RV4S-\\R<S\\R<S\\R<SF\\7SG\\\R<SH\\\7SI\\8SJ\,SK\\9\8\:4SL\ \\\4SM\,SN\ \\\4SO\ \\\4S\\ SP\\;\ \ \9/S4SQ\\8S\ 4>SRjj5r$)[)AnimateDiffVideoToVideoControlNetPipelinea Pipeline for video-to-video generation with ControlNet guidance. This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular device, etc.). The pipeline also inherits the following loading methods: - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters Args: vae ([`AutoencoderKL`]): Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. text_encoder ([`CLIPTextModel`]): Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). tokenizer (`CLIPTokenizer`): A [`~transformers.CLIPTokenizer`] to tokenize text. unet ([`UNet2DConditionModel`]): A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. motion_adapter ([`MotionAdapter`]): A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. controlnet ([`ControlNetModel`] or `List[ControlNetModel]` or `Tuple[ControlNetModel]` or `MultiControlNetModel`): Provides additional conditioning to the `unet` during the denoising process. If you set multiple ControlNets as a list, the outputs from each ControlNet are added together to create one combined additional conditioning. scheduler ([`SchedulerMixin`]): A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. z&text_encoder->image_encoder->unet->vae)feature_extractor image_encodermotion_adapter)r8 prompt_embedsnegative_prompt_embedsNvae text_encoder tokenizerunetrX controlnetrNrVrWc >[T U]5 [U[5(a[R "XE5n[U[ [45(a [U5nURUUUUUUUUU S9 [USS5(a/S[URRR5S- -OSUl[!URS9Ul[!URSSS 9Ulg) N) r[r\r]r^rXr_rNrVrWr[r*r/)vae_scale_factorTF)rbdo_convert_rgb do_normalize)super__init__ isinstancerr from_unet2dlisttuplerregister_modulesgetattrrMr[configblock_out_channelsrbr)video_processorcontrol_video_processor) selfr[r\r]r^rXr_rNrVrWrLs r<rf2AnimateDiffVideoToVideoControlNetPipeline.__init__s&  d0 1 1"..tDD j4- 0 0-j9J %)!/'  W^^bdikoVpVpc$((//*L*L&MPQ&Q Rvw-t?T?TU'5!224V[( $r>rYrZ lora_scale clip_skipc  UbS[U[5(a>Xl[(d[ UR U5 O[ UR U5 Ub[U[[45(aSn O3Ub![U[5(a [U5n OURSn UGc[U[5(aURXR5nURUSURRSSS9n U R n URUSSS 9R n U RS U RS :a["R$"X5(djURR'U SS2URRS- S 245n[(R+S URRS U35 [-UR R.S 5(aAUR R.R0(aU R2R5U5nOSnU c%UR U R5U5US9nUSnOQUR U R5U5USS9nUS U S-*nUR R6R9U5nUR bUR R:nO0UR<bUR<R:nO UR:nUR5UUS9nURunnnUR?SUS5nURAUU-US 5nU(GaUGc|UcS/U -nOUb;[CU5[CU5La$[ES[CU5S[CU5S35e[U[5(aU/nO2U [U5:wa![GSUS[U5SUSU S3 5eUn[U[5(aURUUR5nURSnURUSUSSS9n[-UR R.S 5(aAUR R.R0(aUR2R5U5nOSnUR UR R5U5US9nUSnU(aGURSnUR5UUS9nUR?SUS5nURAX-US 5nUR b6[U[5(a![(a[IUR U5 Xg4$)a, Encodes the prompt into text encoder hidden states. Args: prompt (`str` or `List[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. lora_scale (`float`, *optional*): A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. Nr/r max_lengthTpt)paddingrv truncationreturn_tensorslongest)rxrzz\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: use_attention_mask)attention_mask)r~output_hidden_states)dtyper@z?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.)%rgr _lora_scaler"rr\r%strdictrirMshapermaybe_convert_promptr]model_max_length input_idstorchequal batch_decodeloggerwarningr9rmr}r~to text_modelfinal_layer_normrr^repeatviewtype TypeErrorrEr&)rqpromptr@num_images_per_promptdo_classifier_free_guidancenegative_promptrYrZrsrt batch_size text_inputstext_input_idsuntruncated_ids removed_textr~prompt_embeds_dtypebs_embedseq_len_ uncond_tokensrv uncond_inputs r< encode_prompt7AnimateDiffVideoToVideoControlNetPipeline.encode_prompt!sUV  !j7U&V&V) $#.t/@/@*M!$"3"3Z@  *Vc4["A"AJ  Jvt$<$<VJ&,,Q/J  $ ;<<226>>J..$>>::# )K)22N"nnVYW[n\ffO$$R(N,@,@,DDU[[NN $~~::#At~~'F'F'JR'O$OP  778 ,Q t((//1EFF4K\K\KcKcKvKv!,!;!;!>!>v!F!% $ 1 1.2C2CF2K\j 1 k -a 0 $ 1 1"%%f-ncg!2! !.b 1IM2B C !% 1 1 < < M Mm \    ("&"3"3"9"9  YY ""&))// "/"5"5 %((/B6(R ,22'1%,,Q0EqI %**86K+KWVXY  '+A+I&!#z 1 #V D>$%# *Lt((//1EFF4K\K\KcKcKvKv!-! " &,2215G%;%>%>EXag%>%h "%;%B%B1F[]^%_ "%;%@%@Acelnp%q "    ($ >??DTDT#D$5$5zB44r>cd[URR55Rn[ U[ R 5(dURUSS9RnURX%S9nU(aqURUSS9RSnURUSS9nUR[ R"U5SS9RSnURUSS9nXg4$URU5RnURUSS9n[ R"U5n X4$) Nrw)rzr@rT)rrdim)nextrWrJrrgrTensorrV pixel_valuesr hidden_statesrepeat_interleave zeros_like image_embeds) rqimager@rrrimage_enc_hidden_statesuncond_image_enc_hidden_statesruncond_image_embedss r< encode_image6AnimateDiffVideoToVideoControlNetPipeline.encode_images?T''2245;;%..**5*FSSE4 &*&8&8UY&8&Z&h&hik&l #&=&O&OPekl&O&m #-1-?-?  'd.@.mB. *.L-]-]%1.^. *+J J--e4AAL'99:OUV9WL"'"2"2<"@ 4 4r>c /nU(a/nUGc&[U[5(dU/n[U5[URRR 5:waB[ S[U5S[URRR 5S35e[XRRR 5Hhup[U [5(+n URXSU 5upURU SSS245 U(dMPWRU SSS245 Mj OEUH?n U(a$U RS5upWRU 5 URU 5 MA /n[U5Hwup[R"U /U-SS9n U(a2[R"WU /U-SS9n [R"X/SS9n U RUS9n URU 5 My U$) NzK`ip_adapter_image` must have same length as the number of IP Adapters. Got z images and z IP Adapters.r/r*rr)r@)rgrirMr^encoder_hid_projimage_projection_layersrEziprrappendchunk enumeratercatr)rqip_adapter_imageip_adapter_image_embedsr@rrrnegative_image_embedssingle_ip_adapter_imageimage_proj_layeroutput_hidden_statesingle_image_embedssingle_negative_image_embedsis r<prepare_ip_adapter_image_embedsIAnimateDiffVideoToVideoControlNetPipeline.prepare_ip_adapter_image_embedss9 &$& ! " *.55$4#5 #$DII,F,F,^,^(__ abefvbwaxyEFIJNJSJSJdJdJ|J|F}E~~KL>A ))"<"<"T"T>9'+55E*W&W#DHDUDU+Q8KEA###$7a$@A..)001MdTUg1VW>(?#.H[HaHabcHdE0)001MN##$78 (? #%&/ &= "A"'))-@,ADY,Y_`"a */4yy:OPQ:R9SVk9kqr/s,&+ii1M0cij&k#"5"8"8"8"G  # * *+> ?'>'&r>decode_chunk_sizereturnc/n[S[U5U5H=nXXS-n[URR U5US9nUR U5 M? [ R"U5$)Nr)r2)rangerMr=r[encoderrr)rqvideor2rr8r batch_videos r< encode_video6AnimateDiffVideoToVideoControlNetPipeline.encode_videoscq#e*&78AA$9:K*488??;+GS\]K NN; '9yy!!r>cFSURRR- U-nURup4pVnUR SSSSS5R X5-XFU5n/n[ SURSU5H?n XX-n URRU 5Rn URU 5 MA [R"U5nUSSS24R X5S4URSS-5R SSSSS5nUR5nU$)Nr/rr*rr|) r[rmscaling_factorrpermutereshaperdecoder6rrrfloat) rqr8rrchannels num_framesheightwidthrr batch_latentss r<decode_latents8AnimateDiffVideoToVideoControlNetPipeline.decode_latents(sdhhoo444w>:A--7 j%//!Q1a0889PRZdijq'--*,=>A#(=>M HHOOM:AAM LL '?  % dAg&& 'CekkRSRTo'UV^^_`bcefhiklm  r>cnS[[R"URR5R R 55;n0nU(aX$S'S[[R"URR5R R 55;nU(aXS'U$)Netar2)rFrGrHrNsteprJrK)rqr2r accepts_etaextra_step_kwargsaccepts_generators r<prepare_extra_step_kwargsCAnimateDiffVideoToVideoControlNetPipeline.prepare_extra_step_kwargs;s s7#4#4T^^5H5H#I#T#T#Y#Y#[\\  '*e $(3w/@/@ATAT/U/`/`/e/e/g+hh -6k *  r>?c  ^^US:dUS:a[SU35eUS-S:wd US-S:wa[SUSUS35eU bX[U4SjU 55(d>[S TRS U Vs/sHnUTR;dMUPM sn35eUbU b[S US U S 35eUcU c [S5eUb7[U[[ [ 45(d[S[U535eUbU b[SUSU S 35eU bCU b@U RU R:wa&[SU RSU RS35eUbUb [S5eU bU b [S5eU b[[U [ 5(d[S[U 535eU SRS;a[SU SRS35e[TR[5(aW[U[ 5(aB[RS[TRR5S[U5S35 [!["S5=(a8 [TR[$R&R(R*5nUc [U5OURSn[TR[,5(d0U(a[TRR.[,5(aW[T[ 5(d[1S[T535e[T5U:wa[S US![T5<35eO[TR[5(d0U(a[TRR.[5(a[T[ 5(a[TS[ 5(d[1S"[T5<35e[TS5U:wa[S#US![T5<35e[3U4S$jT55(a [S%5eOe[TR[,5(d0U(aJ[TRR.[,5(a![U[45(d [1S&5eO[TR[5(d0U(a[TRR.[5(a[U[ 5(a#[3S'U55(a [S(5eOO[U[ 5(a7[U5[TRR5:wa [S)5eOe[U[6[ 45(dU/n[U[6[ 45(dU/n[U5[U5:wa$[S*[U5S+[U5S,35e[TR[5(a[U5[TRR5:wa[[S-US.[U5S/[TRR5S0[TRR5S3 5e[9UU5HHunnUU:a[S1US2US35eUS3:a[S1US435eUS5:dM;[S6US735e gs snf)8Nrr/z2The value of strength should in [0.0, 1.0] but is raz7`height` and `width` have to be divisible by 8 but are z and rc3@># UHoTR;v M g7fN)_callback_tensor_inputs).0krqs r< IAnimateDiffVideoToVideoControlNetPipeline.check_inputs..esF 7Y!-- -7Ysz2`callback_on_step_end_tensor_inputs` has to be in z , but found zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z:`prompt` has to be of type `str`, `list` or `dict` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` z3Only one of `video` or `latents` should be providedzProvide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined.z:`ip_adapter_image_embeds` has to be of type `list` but is )rrzF`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is Dz You have z! ControlNets and you have passed z= prompts. The conditionings will be fixed across the prompts.scaled_dot_product_attentionr*z>For single controlnet, `image` must be of type `list` but got zExcepted image to have length z" but got len(conditioning_frames)=z_For multiple controlnets: `image` must be type list of lists but got type(conditioning_frames)=z$Expected length of image sublist as c3X># UHn[U5[TS5:gv M! g7f)rN)rM)rimgconditioning_framess r<rrs'ZFYs3s8s#6q#9::FYs'*zDAll conditioning frame batches for multicontrolnet must be same sizezLFor single controlnet: `controlnet_conditioning_scale` must be type `float`.c3B# UHn[U[5v M g7fr)rgri)rrs r<rrsR4Qqz!T**4QszEA single batch of multiple conditionings are supported at the moment.zFor multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have the same length as the number of controlnetsz`control_guidance_start` has z* elements, but `control_guidance_end` has zI elements. Make sure to provide the same number of elements to each list.z`control_guidance_start`: z has z elements but there are z- controlnets available. Make sure to provide zcontrol guidance start: z4 cannot be larger or equal to control guidance end: rz can't be smaller than 0.rzcontrol guidance end: z can't be larger than 1.0.)rEallrrgrrirrrndimr_rrrrMnetsr9Fr_dynamo eval_frameOptimizedModuler _orig_modranyrrjr)rqrstrengthrrrrr8rrYrZrr"callback_on_step_end_tensor_inputscontrolnet_conditioning_scalecontrol_guidance_startcontrol_guidance_endr is_compiledrstartends` ` r< check_inputs6AnimateDiffVideoToVideoControlNetPipeline.check_inputsLs& a<8a<QRZQ[\] ] A:?eai1nVW]V^^cdicjjklm m - 9#F 7YF C C DTEaEaDbbn|^pH|^vwbckokGkGbGpq|^pHoIJ   -";08N}o^00 ^ 5w   6Ct;L(M(MYZ^_eZfYghi i  &+A+M9/9JK*++]_   $)?)K""&<&B&BB --:-@-@,AB.445Q8  !4RS S  ',C,O^  # .5t<< PQUVmQnPop)+00> \]tuv]w]|]|\}}~ doo'; < <&$''DOO$8$8 9::[\_`f\g[hSTa!?@ Z OOU]]55EEF $+?SZ a8H  t 8 84??44oFF1488TUYZmUnTop&':5 #A*MpWZ[nWoVq!rss6 t(< = =4??446JKK1488 K^_`Kacg@h@hv\`at\u[wx&q)*j8 ::,FiPSTgPhOjkZFYZZZ !ghh[ 5 t 8 84??44oFF;UCC nooD t(< = =4??446JKK7>>R4QRRR$%lmmS94@@SIfEgkn$$lF!D 505$-@@&<%= ". >>$8#9 % &#.B*C C/4J0K/LLvwz|PxQwRR[\  doo'; < <)*c$//2F2F.GG 01G0HcRhNiMjkCDGHLHWHWH\H\D]C^^KLOPTP_P_PdPdLeKffgh46JKJE3| .ug5ijminnops{ #;E7B[!\]]Sy #9#>X!YZZLGpHs 5\  \ c[[X-5U5n[X- S5nX&URR-SnX!U- 4$)Nr)minintmaxrNorder)rqr?rArr@ init_timestept_starts r< get_timesteps7AnimateDiffVideoToVideoControlNetPipeline.get_timestepssPC 3 >?ATU )91=(<(<<>? 777r>r/Frrrnum_channels_latentsrtimesteprr@r2r8 add_noisec U cURSOU RSn UUU X R-X0R-4n[U [5(a*[ U 5U:wa[ S[ U 5SUS35eU GcZUR RR(a7UR5nUR R[RS9 [U [5(a@[U5Vs/sH)nURXXU 5RS5PM+ nnO1UVs/sH$nURUX5RS5PM& nn[R "USS9nUR RR(aUR RU5 URU5nUR RR"U-nUURS:a7UURS-S:Xa!S US URSS 3n[ U5eUURS:a5UURS-S:wa[ S URSS US35e[R "U/SS9n[%URXUS9nUR&R)UUU5R+SSSSS5n U $XR:wa[ SU<SU R<35eU RXS9n U (a'[%XXS9nUR&R)U UU5n U $s snfs snf)Nr/r*z/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.rrrzYou have passed z# text prompts (`prompt`), but only zp initial images (`image`). Please make sure to update your script to pass as many initial images as text promptsz'Cannot duplicate `image` of batch size z to z text prompts.)r2r@rrrz!`latents` expected to have shape=z, but found latents.shape=)rrbrgrirMrEr[rm force_upcastrrrfloat32rr unsqueezerrr(rNrr)rqrrrrrrrr@r2r8rrrrr init_latentsvid error_messagenoises r<prepare_latents9AnimateDiffVideoToVideoControlNetPipeline.prepare_latentssF(/U[[^GMM!Z+GA#i.AQR&<'gi  ?xx++  %-- 0)T**#:. .%%eh >OPZZ[\].  nssmrfi 1 1#y T ^ ^_` amr s 99\q9Lxx++ E"'??51L88??99LHLL..q11jP>PQR>S=TTXYcXddrs %yy,Q?  !3!3y_deEnn..|UHMUUVWYZ\]_`bcdG % #EuhFaSZS`S`Rb!cddjjj5G$U\..227E8LW  ts 0M 9+Mc |URRXUS9R[RS9nUR SSSSS5R SS5nURSn U S:XaUn OUn URU SS9nURXgS 9nU(a!U (d[R"U/S-5nU$) Nrrrrr*r/rrrr) rppreprocess_videorrrrflattenrrr) rqrrrrnum_videos_per_promptr@rr guess_modevideo_batch_size repeat_bys r<prepare_conditioning_framesEAnimateDiffVideoToVideoControlNetPipeline.prepare_conditioning_framesTs,,==eZ_=`cc--d  aAq!,44Q: ;;q> q "I.I'' q'94 &zIIugk*E r>cUR$r_guidance_scalerqs r<guidance_scale8AnimateDiffVideoToVideoControlNetPipeline.guidance_scalets###r>cUR$r) _clip_skipr3s r<rt3AnimateDiffVideoToVideoControlNetPipeline.clip_skipx r>c URS:$)Nr/r1r3s r<rEAnimateDiffVideoToVideoControlNetPipeline.do_classifier_free_guidances##a''r>cUR$r)_cross_attention_kwargsr3s r<cross_attention_kwargs@AnimateDiffVideoToVideoControlNetPipeline.cross_attention_kwargss+++r>cUR$r)_num_timestepsr3s r< num_timesteps7AnimateDiffVideoToVideoControlNetPipeline.num_timestepss"""r>cUR$r) _interruptr3s r< interrupt3AnimateDiffVideoToVideoControlNetPipeline.interruptr9r>2g@g?pilTrr?enforce_inference_stepsrArBr4rrr*rrrr output_type return_dictr>rr+rr callback_on_step_endrc [UR5(aURRO URn [U[5(d%[U[5(a[ U5U/-nO[U[5(d%[U[5(a[ U5U/-nOb[U[5(dM[U[5(d8[U [ 5(a[ U R5OSn!U!U/-U!U/-nnU=(d- URRRUR-nU=(d- URRRUR-nSn URUU UUU UUUUUUUUUUUS9 Xl UUlUUlSUlUb[U["[$45(aSn"O3Ub![U[5(a [ U5n"OUR&Sn"UR(n#UR*n$U(dG[-UR.UU#Xx5upuUR1XWU U#5upuUSSR3U"U -5n%OD[5XZ- 5n&[-UR.U&U#Xx5unn&Xu*SnUSSR3U"U -5n%Uc?UR6R9XUS9nUR;SSSSS 5nUR=U#U$S 9nURRR>n'URAUUUU'U"U -U%U$U#UUUUS 9 nURBbURBRES S5OSn(UR&Sn)URF(a0URIUU)U#U URJU UUU(URLS 9 unnOgUROUU#U URJU UUU(URLS9 unnURJ(a[PRR"UU/5nURUU)SS9nUcUb"URWUUU#U"U -URJ5n*[U [ 5(a.[U[X5(aU/[ U R5-n[U [Z5(aU RR\O"U RSRR\n+U=(d U+n/n,[_[ U55Hn-[aUU5V.V/s/sH>un.n/S[YU-[ U5- U.:=(d U-S-[ U5- U/:5- PM@ n0n.n/U,Rc[U [Z5(aU0SOU05 M [U [Z5(a2UReUUUU"U -U)-U U#U R*URJUS9 nOg[U [ 5(aP/n1UHEn2UReU2UUU"U -U)-U U#U R*URJUS9 n3U1RcU35 MG U1nOeURgX5n4UcUbSW*0OSn5URh(a URjOSn6[_U65GHn7URh(aBURmUU7UU#UR*U5up[ U5nUR1XWU U#5upu[ U5Ul7[ U5XPR.Rp-- n8URsURnS9n9[uU5GHun-n:URv(aMURJ(a[PRR"U/S-5OUn;UR.RyU;U:5n;U(aDURJ(a3UnV.s/sH un>n.U>U.-PM n?n>n.O$Un@[U@[5(aW@Sn@W@U,U--n?[PR|"Uf!,(df  GM=f)u The call function to the pipeline for generation. Args: video (`List[PipelineImageInput]`): The input video to condition the generation on. Must be a list of images/frames of the video. prompt (`str` or `List[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The width in pixels of the generated video. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality videos at the expense of slower inference. timesteps (`List[int]`, *optional*): Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. Must be in descending order. sigmas (`List[float]`, *optional*): Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed will be used. strength (`float`, *optional*, defaults to 0.8): Higher strength leads to more differences between original video and generated video. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. generator (`torch.Generator` or `List[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor is generated by sampling using the supplied random `generator`. Latents should be of shape `(batch_size, num_channel, num_frames, height, width)`. prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, text embeddings are generated from the `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. conditioning_frames (`List[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If multiple ControlNets are specified, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated video. Choose between `torch.Tensor`, `PIL.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`AnimateDiffPipelineOutput`] instead of a plain tuple. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. callback_on_step_end (`Callable`, *optional*): A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. callback_on_step_end_tensor_inputs (`List`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. decode_chunk_size (`int`, defaults to `16`): The number of frames to decode at a time when calling `decode_latents` method. Examples: Returns: [`pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] or `tuple`: If `return_dict` is `True`, [`pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] is returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. r/)rrrrrrYrZrrr8rrrrrr FNrr'r*rrr) rrrrrrrr@r2r8rrscale) rrr@r*rrrYrZrsrt)rYrZrsrt)repeatsrr) rrrrr*r@rrr+r)totalr|)encoder_hidden_statescontrolnet_condconditioning_scaler+rL)rRr>added_cond_kwargsdown_block_additional_residualsmid_block_additional_residualr8rYrZlatent)rrK)frames)Mr'r_rrgrirMrrr^rm sample_sizerbr r2r7r=rErrr_execution_devicerrRrNrrrror(rr in_channelsr$r>getfree_noise_enabled_encode_prompt_free_noiserrtrrrrrrrglobal_pool_conditionsrrrr.rfree_init_enabled_free_init_num_iters_apply_free_initrAr progress_barrrFscale_model_inputr transposerr6r prev_samplelocalspopupdate XLA_AVAILABLExm mark_steprpostprocess_videomaybe_free_model_hooksr0)Jrqrrrrr?rJrArBr4rrr*rr2r8rYrZrrrrKrLr>rr+rr rtrMrrr_multrr@rlatent_timestepdenoising_inference_stepsrtext_encoder_lora_scalerrr`controlnet_keeprsekeepscond_prepared_videosframe_prepared_videorrUnum_free_init_itersfree_init_iternum_warmup_stepsrdtlatent_model_inputcontrol_model_inputcontrolnet_prompt_embedsc cond_scalecontrolnet_cond_scaledown_block_res_samplesmid_block_res_sample noise_prednoise_pred_uncondnoise_pred_textcallback_kwargsrcallback_outputs video_tensorsJ r<__call__2AnimateDiffVideoToVideoControlNetPipeline.__call__s N3ET__2U2UT__..[_[j[j 0$77JG[]aBXAY%Y "0$77JG]_cBVAW#W 2D99*MacgBhBh+5jBV+W+W3z']^D.//,--%9 " O499++77$:O:OOM))558M8MM ! +'#9 3-$;/Q*G#9!5!  & .#'=$  *Vc4["A"AJ  Jvt$<$<VJ&,,Q/J'' '-? 3VY. *I.2-?-?@S`hjp-q *I'm22:@U3UVO(+,?,J(K %3E 9694 0I0""6"78I'm22:@U3UVO ?((99%V[9\EMM!Q1a0EHHF%H8E#yy//;;&&!5!$99$/-'  "?C>Y>Y>eD ' ' + +GT :ko ]]1%  " "484R4R%&;,0,L,L /+'=2..5S 5 1M1594F4F%00+'=2..5G 5 1M1// % +A=*Q R );;JTU;VM  '+B+N?? '2200 L j"6 7 7JGdfk=9v ( "E..w8IJL((::[f:gE ##%8O(66o`&p'>=s-AiC(i(=i" G i(i("i(( i8 ) r7r=r2rErrArprbro)NN)NNNNNr)r) NNNNNNNNNrrr) N@rrr/NNNNNrF)FF)?__name__ __module__ __qualname____firstlineno____doc__model_cpu_offload_seq_optional_componentsrrr r r rrrrrrrrr!r rrrr r rfrrrrrrrrrrrr rrr@ Generatorboolr$r.propertyr4rtrr>rBrFno_gradrrrrrr__static_attributes__ __classcell__)rLs@r<rTrTs@EST$157;#) ) $) ! ) (/9: ) & ) /4+@%BXZnno)    " + '  ) ) .!) "5#) ) d049=&*#'t5  -t5!) 6t5UOt5C=t5n52+'\""U\\"&!. # $+/&)" #h[V8)-$%"&'+)-MQ*.!#P %PP P " P  P3-P $P&PE%//43H"HIJP%,,'PPP Px%*@$$ ((,,## ]]_1526 $##%(-)-(, #;?/0MQ*.049=9=@DBF%* ;?CF ?+y7,c]-y7./y70!)c3h 81y72(-UDK-?'@3y745y76!&eT%[&8 97y78$E4;$679y7:C=;y7<'xc40@$0F'GH=y7>-1I?y7@Ay7y7r>rT)Nr6)NNNN)RrGtypingrrrrrrr rtorch.nn.functionalnn functionalr transformersr r r r image_processorrloadersrrrrmodelsrrrrrr models.lorarmodels.unets.unet_motion_modelr schedulersrrrrr r!utilsr"r#r$r%r&utils.torch_utilsr'r(ror)free_init_utilsr+free_noise_utilsr,pipeline_utilsr-r.pipeline_outputr0torch_xla.core.xla_modelcore xla_modelrlrk get_loggerrrEXAMPLE_DOC_STRINGrrrr=rr@rrRrTrDr>r<rsRDDD hh1ww:;onA-+8D6))MM   H %=Dck TLL T-5eoo-F T\_ T *.15%)$( 8*!#8* U3 ,- .8*S " 8* T%[ ! 8*vB7"B7r>