
    h              0       :   S SK r S SKrS SKrS SKJr  S SKJrJrJrJ	r	  S SK
rS SKrS SKJs  Jr  S SKJr  S SKJrJrJr  S SKJr  S SKJr  SS	KJr  S
SKJr  \" 5       (       a  S SKJr  S SK J!r!  \RD                  " \#5      r$\ " S S5      5       r%\ " S S5      5       r& " S S5      r'S r(S r)    S8S\RT                  S\+S\RT                  S\,S\-S\\RT                     S\&4S jjr.\R^                  " 5                         S9S\\	\0\1\0   4      S\\+   S \\+   S!\+S"\,S#\\	\0\1\0   4      S$\\+   S\,S%\\	\Rd                  \1\Rd                     4      S&\\RT                     S'\\RT                     S(\\RT                     S)\\0   S*\-S+\\\+\+\RT                  /S4      S,\+S-\\3\0\4      S.\,4$S/ jj5       r4                       S:S\\	\0\1\0   4      S\\+   S \\+   S!\+S"\,S0\-S1\-S2\-S3\+S4\5S#\\	\0\1\0   4      S$\\+   S\,S%\\	\Rd                  \1\Rd                     4      S&\\RT                     S'\\RT                     S(\\RT                     S)\\0   S*\-S+\\\+\+\RT                  /S4      S,\+S-\\3\0\4      S.\,4.S5 jjr6 " S6 S7\'5      r7g);    N)	dataclass)AnyCallableOptionalUnion)logging)DDIMSchedulerStableDiffusionPipelineUNet2DConditionModel)rescale_noise_cfg)is_peft_available   )randn_tensor   )convert_state_dict_to_diffusers)
LoraConfig)get_peft_model_state_dictc                   t    \ rS rSr% Sr\R                  \S'   \R                  \S'   \R                  \S'   Srg)DDPOPipelineOutput)   a;  
Output class for the diffusers pipeline to be finetuned with the DDPO trainer

Args:
    images (`torch.Tensor`):
        The generated images.
    latents (`list[torch.Tensor]`):
        The latents used to generate the images.
    log_probs (`list[torch.Tensor]`):
        The log probabilities of the latents.

imageslatents	log_probs N	__name__
__module____qualname____firstlineno____doc__torchTensor__annotations____static_attributes__r       U/home/james-whalen/.local/lib/python3.13/site-packages/trl/models/modeling_sd_base.pyr   r   )   s'     LL\\||r%   r   c                   V    \ rS rSr% Sr\R                  \S'   \R                  \S'   Srg)DDPOSchedulerOutput=   aH  
Output class for the diffusers scheduler to be finetuned with the DDPO trainer

Args:
    latents (`torch.Tensor`):
        Predicted sample at the previous timestep. Shape: `(batch_size, num_channels, height, width)`
    log_probs (`torch.Tensor`):
        Log probability of the above mentioned sample. Shape: `(batch_size)`
r   r   r   Nr   r   r%   r&   r(   r(   =   s     \\||r%   r(   c                       \ rS rSrSrS\4S jrS\4S jr\	S 5       r
\	S 5       r\	S 5       r\	S	 5       r\	S
 5       r\	S 5       rS rS rS rS rS rSrg)DDPOStableDiffusionPipelineM   zM
Main class for the diffusers pipeline to be finetuned with the DDPO trainer
returnc                     [         eNNotImplementedErrorselfargskwargss      r&   __call__$DDPOStableDiffusionPipeline.__call__R       !!r%   c                     [         er/   r0   r2   s      r&   scheduler_step*DDPOStableDiffusionPipeline.scheduler_stepU   r8   r%   c                     [         e)z0
Returns the 2d U-Net model used for diffusion.
r0   r3   s    r&   unet DDPOStableDiffusionPipeline.unetX   
    
 "!r%   c                     [         e)za
Returns the Variational Autoencoder model used from mapping images to and from the latent space
r0   r=   s    r&   vaeDDPOStableDiffusionPipeline.vae_   r@   r%   c                     [         e)z7
Returns the tokenizer used for tokenizing text inputs
r0   r=   s    r&   	tokenizer%DDPOStableDiffusionPipeline.tokenizerf   r@   r%   c                     [         e)zS
Returns the scheduler associated with the pipeline used for the diffusion process
r0   r=   s    r&   	scheduler%DDPOStableDiffusionPipeline.schedulerm   r@   r%   c                     [         e)z8
Returns the text encoder used for encoding text inputs
r0   r=   s    r&   text_encoder(DDPOStableDiffusionPipeline.text_encodert   r@   r%   c                     [         e)z&
Returns the autocast context manager
r0   r=   s    r&   autocast$DDPOStableDiffusionPipeline.autocast{   r@   r%   c                     [         e)z/
Sets the progress bar config for the pipeline
r0   r2   s      r&   set_progress_bar_config3DDPOStableDiffusionPipeline.set_progress_bar_config   
     "!r%   c                     [         e)z 
Saves all of the model weights
r0   r2   s      r&   save_pretrained+DDPOStableDiffusionPipeline.save_pretrained   rS   r%   c                     [         e)z2
Returns the trainable parameters of the pipeline
r0   r2   s      r&   get_trainable_layers0DDPOStableDiffusionPipeline.get_trainable_layers   rS   r%   c                     [         e)za
Light wrapper around accelerate's register_save_state_pre_hook which is run before saving state
r0   r2   s      r&   save_checkpoint+DDPOStableDiffusionPipeline.save_checkpoint   rS   r%   c                     [         e)za
Light wrapper around accelerate's register_lad_state_pre_hook which is run before loading state
r0   r2   s      r&   load_checkpoint+DDPOStableDiffusionPipeline.load_checkpoint   rS   r%   r   N)r   r   r   r   r    r   r6   r(   r:   propertyr>   rB   rE   rH   rK   rN   rQ   rU   rX   r[   r^   r$   r   r%   r&   r+   r+   M   s    "+= ""1D " " " " " " " " " " " " """"""r%   r+   c                     U R                   nU[        U5      :  a  [        S5      eU R                  U R                  S[        U5      U-
  -  -   5      R                  U5      $ )z
As opposed to the default direction of broadcasting (right to left), this function broadcasts from left to right

    Args:
        input_tensor (`torch.FloatTensor`): is the tensor to broadcast
        shape (`tuple[int]`): is the shape to broadcast to
zrThe number of dimensions of the tensor to broadcast cannot be greater than the length of the shape to broadcast to)r   )ndimlen
ValueErrorreshapeshapebroadcast_to)input_tensorrf   
input_ndims      r&   _left_broadcastrj      sf     ""JCJ A
 	
  2 2TSZ*=T5U UVccdijjr%   c                    [         R                  " U R                  SUR                  5       5      R	                  UR
                  5      n[         R                  " UR                  5       S:  U R                  R                  SUR                  5       5      U R                  5      R	                  UR
                  5      nSU-
  nSU-
  nXe-  SX4-  -
  -  nU$ )Nr   r   )r!   gatheralphas_cumprodcputodevicewherefinal_alpha_cumprod)r3   timestepprev_timestepalpha_prod_talpha_prod_t_prevbeta_prod_tbeta_prod_t_prevvariances           r&   _get_variancerz      s    << 3 3QGJJ8??[Lq ""1m&7&7&9:   	b	 
 l"K,, .1|7W3WXHOr%   model_outputrs   sampleetause_clipped_model_outputprev_sampler-   c           	      D   U R                   c  [        S5      eX R                  R                  U R                   -  -
  n[        R
                  " USU R                  R                  S-
  5      nU R                  R                  SUR                  5       5      n	[        R                  " UR                  5       S:  U R                  R                  SUR                  5       5      U R                  5      n
[        XR                  5      R                  UR                  5      n	[        XR                  5      R                  UR                  5      n
SU	-
  nU R                  R                  S:X  a  X;S-  U-  -
  U	S-  -  nUnOU R                  R                  S:X  a  UnX9S-  U-  -
  US-  -  nO`U R                  R                  S:X  a#  U	S-  U-  US-  U-  -
  nU	S-  U-  US-  U-  -   nO#[        SU R                  R                   S	35      eU R                  R                   (       a  U R#                  U5      nOVU R                  R$                  (       a;  UR                  U R                  R&                  * U R                  R&                  5      n[)        XU5      nXNS-  -  n[        XR                  5      R                  UR                  5      nU(       a  X9S-  U-  -
  US-  -  nSU
-
  US
-  -
  S-  U-  nU
S-  U-  U-   nUb  Ub  [        S5      eUc2  [+        UR                  UUR                  UR,                  S9nUUU-  -   nUR/                  5       U-
  S
-  * S
US
-  -  -  [        R0                  " U5      -
  [        R0                  " [        R2                  " S
[        R4                  " [6        R8                  5      -  5      5      -
  nUR;                  [=        [?        SUR@                  5      5      S9n[C        URE                  UR,                  5      U5      $ )a  

Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion process
from the learned model outputs (most often the predicted noise).

Args:
    self: scheduler.
    model_output (`torch.FloatTensor`): direct output from learned diffusion model.
    timestep (`int`): current discrete timestep in the diffusion chain.
    sample (`torch.FloatTensor`):
        current instance of sample being created by diffusion process.
    eta (`float`): weight of noise for added noise in diffusion step.
    use_clipped_model_output (`bool`): if `True`, compute "corrected" `model_output` from the clipped
        predicted original sample. Necessary because predicted original sample is clipped to [-1, 1] when
        `self.config.clip_sample` is `True`. If no clipping has happened, "corrected" `model_output` would coincide
        with the one provided as input and `use_clipped_model_output` will have not effect.
    generator: random number generator.
    prev_sample (`torch.FloatTensor`, *optional*): if not `None`, the previous sample to be used

Returns:
    `DDPOSchedulerOutput`: the predicted sample at the previous timestep and the log probability of the sample
zaNumber of inference steps is 'None', you need to run 'set_timesteps' after creating the schedulerr   r   epsilong      ?r|   v_predictionzprediction_type given as z6 must be one of `epsilon`, `sample`, or `v_prediction`r   zsCannot pass both generator and prev_sample. Please make sure that either `generator` or `prev_sample` stays `None`.)	generatorrp   dtype)dim)#num_inference_stepsrd   confignum_train_timestepsr!   clamprm   rl   rn   rq   rr   rj   rf   ro   rp   prediction_typethresholding_threshold_sampleclip_sampleclip_sample_rangerz   r   r   detachlogsqrt	as_tensornppimeantuplerangerb   r(   type)r3   r{   rs   r|   r}   r~   r   r   rt   ru   rv   rw   pred_original_samplepred_epsilonry   	std_dev_tpred_sample_directionprev_sample_meanvariance_noiselog_probs                       r&   r:   r:      s   B 'o
 	
 {{>>$BZBZZZMKKq$++2Q2QTU2UVM &&--a@Lq ""1m&7&7&9:  
 #<>AA&--PL'(9<<HKKFMMZl"K {{""i/ &)=)L LP\adPee#		$	$	0+#!69M!MMQ\adQee		$	$	6 ,c 1V;{C?OS_>__$c)\9[#=MQW<WW'(C(C'D E 
 	
 {{#556JK		 	 399[[***DKK,I,I 
 T];H3''I	<<8;;FMMJI#!69M!MMQ\adQee !22Y\AsKlZ )S14HHK``9#8+
 	

 %&&$$	
 '^)CC "22q
89Q)Q,=OP
))I
	
))EJJq5??255#99:
;	<  }}uQ'>!?}@H{//=xHHr%   promptheightwidthr   guidance_scalenegative_promptnum_images_per_promptr   r   prompt_embedsnegative_prompt_embedsoutput_typereturn_dictcallbackcallback_stepscross_attention_kwargsguidance_rescalec                 x   U=(       d-    U R                   R                  R                  U R                  -  nU=(       d-    U R                   R                  R                  U R                  -  nU R	                  UUUUUUU5        Ub  [        U[        5      (       a  SnO3Ub!  [        U[        5      (       a  [        U5      nOUR                  S   nU R                  nUS:  nUb  UR                  SS5      OSnU R                  UUUUUUUUS9nU R                  R                  UUS9  U R                  R                  nU R                   R                  R                   nU R#                  UU-  UUUUR$                  UU	U
5      n
[        U5      X@R                  R&                  -  -
  nU
/n/ nU R)                  US9 n[+        U5       GHY  u  nnU(       a  [,        R.                  " U
/S	-  5      OU
nU R                  R1                  UU5      nU R                  UUUUS
S9S   n U(       a  U R3                  S	5      u  n!n"U!UU"U!-
  -  -   n U(       a  US:  a  [5        U W"US9n [7        U R                  U UX5      n#U#R8                  n
U#R:                  n$UR=                  U
5        UR=                  U$5        U[        U5      S-
  :X  d/  US-   U:  d  GM
  US-   U R                  R&                  -  S:X  d  GM-  UR?                  5         Uc  GMC  UU-  S:X  d  GMO  U" UUU
5        GM\     SSS5        US:X  d^  U R@                  RC                  XR@                  R                  RD                  -  S
S9S   n%U RG                  U%UUR$                  5      u  n%n&OU
n%Sn&U&c  S/U%R                  S   -  n'OU& V(s/ s H
  n(U((       + PM     n'n(U RH                  RK                  U%UU'S9n%[M        U S5      (       a'  U RN                  b  U RN                  RQ                  5         [S        U%UU5      $ ! , (       d  f       GN= fs  sn(f )u  
Function invoked when calling the pipeline for generation. Args: prompt (`str` or `list[str]`, *optional*): The
prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. height
(`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): The height in pixels of the
generated image.
    width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
        The width in pixels of the generated image.
    num_inference_steps (`int`, *optional*, defaults to 50):
        The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense
        of slower inference.
    guidance_scale (`float`, *optional*, defaults to 7.5):
        Guidance scale as defined in [Classifier-Free Diffusion
        Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. of
        [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
        `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the
        text `prompt`, usually at the expense of lower image quality.
    negative_prompt (`str` or `list[str]`, *optional*):
        The prompt or prompts not to guide the image generation. If not defined, one has to pass
        `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
        less than `1`).
    num_images_per_prompt (`int`, *optional*, defaults to 1):
        The number of images to generate per prompt.
    eta (`float`, *optional*, defaults to 0.0):
        Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies
        to [`schedulers.DDIMScheduler`], will be ignored for others.
    generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
        One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to
        make generation deterministic.
    latents (`torch.FloatTensor`, *optional*):
        Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
        generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
        tensor will ge generated by sampling using the supplied random `generator`.
    prompt_embeds (`torch.FloatTensor`, *optional*):
        Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
        provided, text embeddings will be generated from `prompt` input argument.
    negative_prompt_embeds (`torch.FloatTensor`, *optional*):
        Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
        If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument.
    output_type (`str`, *optional*, defaults to `"pil"`):
        The output format of the generate image. Choose between [PIL](https://pillow.readthedocs.io/en/stable/):
        `PIL.Image.Image` or `np.array`.
    return_dict (`bool`, *optional*, defaults to `True`):
        Whether to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a plain tuple.
    callback (`Callable`, *optional*):
        A function that will be called every `callback_steps` steps during inference. The function will be called
        with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
    callback_steps (`int`, *optional*, defaults to 1):
        The frequency at which the `callback` function will be called. If not specified, the callback will be
        called at every step.
    cross_attention_kwargs (`dict`, *optional*):
        A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
        `self.processor` in
        [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py).
    guidance_rescale (`float`, *optional*, defaults to 0.7):
        Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
        Flawed](https://huggingface.co/papers/2305.08891) `guidance_scale` is defined as `φ` in equation 16. of
        [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891).
        Guidance rescale factor should fix overexposure when using zero terminal SNR.

Examples:

Returns:
    `DDPOPipelineOutput`: The generated image, the predicted latents used to generate the image and the associated
    log probabilities
Nr   r         ?scaler   r   
lora_scalerp   totalr   Fencoder_hidden_statesr   r           r   latentr   Tr   do_denormalizefinal_offload_hook)*r>   r   sample_sizevae_scale_factorcheck_inputs
isinstancestrlistrc   rf   _execution_deviceget_encode_promptrH   set_timesteps	timestepsin_channelsprepare_latentsr   orderprogress_bar	enumerater!   catscale_model_inputchunkr   r:   r   r   appendupdaterB   decodescaling_factorrun_safety_checkerimage_processorpostprocesshasattrr   offloadr   ))r3   r   r   r   r   r   r   r   r}   r   r   r   r   r   r   r   r   r   r   
batch_sizerp   do_classifier_free_guidancetext_encoder_lora_scaler   num_channels_latentsnum_warmup_stepsall_latentsall_log_probsr   itlatent_model_input
noise_prednoise_pred_uncondnoise_pred_textscheduler_outputr   imagehas_nsfw_conceptr   has_nsfws)                                            r&   pipeline_stepr   G  sE   p Ktyy''33d6K6KKFITYY%%11D4I4IIE 	 j55
		
64 8 8[
"((+
##F #13"6 LbKm488$Gsw''##5* ( 	M 	NN  !4V D((I  99++77""**	G 9~(;nn>R>R(RR)KM			!4		5i(DAq=XG9q=!9^e!%!A!ABTVW!X "&3'=! #  J +5?5E5Ea5H2!?.?UfCf1gg
*/?#/E.z?]mn
  .dnnj!WZ&..G'11Hw'  * C	NQ&&AE5E+E1q5TXTbTbThThJhlmJm##%'A,>!,CQ7+G ) 
6L ("((//*H*H HV[\]^_"&"9"9%I\I\"]%++a.07GH7G8h,7GH  ,,U\j,kE t)**t/F/F/R'')e[-@@q 
6	5` Is+   =DP%P%6P%P%P%2P7%
P4truncated_backproptruncated_backprop_randgradient_checkpointtruncated_backprop_timesteptruncated_rand_backprop_minmaxc                 	   U=(       d-    U R                   R                  R                  U R                  -  nU=(       d-    U R                   R                  R                  U R                  -  n[        R
                  " 5          U R                  UUUUUUU5        Ub  [        U[        5      (       a  SnO3Ub!  [        U[        5      (       a  [        U5      nOUR                  S   nU R                  nUS:  nUb  UR                  SS5      OSnU R                  UUUUUUUUS9nU R                  R!                  UUS9  U R                  R"                  nU R                   R                  R$                  nU R'                  UU-  UUUUR(                  UUU5      nSSS5        [        W5      X@R                  R*                  -  -
  nU/n/ n U R-                  US9 n![/        U5       GH  u  n"n#W(       a  [        R0                  " U/S	-  5      OUn$U R                  R3                  U$U#5      n$U(       a'  [4        R4                  " U R                   U$U#UUS
S9S   n%OU R                  U$U#UUS
S9S   n%U(       aQ  U(       a4  [6        R8                  " U
S   U
S   5      n&U"U&:  a  U%R;                  5       n%OU"U	:  a  U%R;                  5       n%U(       a  U%R=                  S	5      u  n'n(U'UU(U'-
  -  -   n%U(       a  US:  a  [?        U%W(US9n%[A        U R                  U%U#X5      n)U)RB                  nU)RD                  n*URG                  U5        U RG                  U*5        U"[        U5      S-
  :X  d/  U"S-   U:  d  GM  U"S-   U R                  R*                  -  S:X  d  GM  U!RI                  5         Uc  GM  U"U-  S:X  d  GM  U" U"U#U5        GM     SSS5        US:X  d^  U RJ                  RM                  XRJ                  R                  RN                  -  S
S9S   n+U RQ                  U+WUR(                  5      u  n+n,OUn+Sn,U,c  S/U+R                  S   -  n-OU, V.s/ s H
  n.U.(       + PM     n-n.U RR                  RU                  U+UU-S9n+[W        U S5      (       a'  U RX                  b  U RX                  R[                  5         []        U+UU 5      $ ! , (       d  f       GN5= f! , (       d  f       GN= fs  sn.f )u  
Function to get RGB image with gradients attached to the model weights.

Args:
    pipeline (`StableDiffusionPipeline`): Pipeline to be used for image generation.
    prompt (`str` or `list[str]`, *optional*, defaults to `None`):
        The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`
        instead.
    height (`int`, *optional*, defaults to `pipeline.unet.config.sample_size * pipeline.vae_scale_factor`):
        The height in pixels of the generated image.
    width (`int`, *optional*, defaults to `pipeline.unet.config.sample_size * pipeline.vae_scale_factor`):
        The width in pixels of the generated image.
    num_inference_steps (`int`, *optional*, defaults to `50`):
        The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense
        of slower inference.
    guidance_scale (`float`, *optional*, defaults to `7.5`):
        Guidance scale as defined in [Classifier-Free Diffusion
        Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. of
        [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting
        `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the
        text `prompt`, usually at the expense of lower image quality.
    truncated_backprop (`bool`, *optional*, defaults to True):
        Truncated Backpropation to fixed timesteps, helps prevent collapse during diffusion reward training as
        shown in AlignProp (https://huggingface.co/papers/2310.03739).
    truncated_backprop_rand (`bool`, *optional*, defaults to True):
        Truncated Randomized Backpropation randomizes truncation to different diffusion timesteps, this helps
        prevent collapse during diffusion reward training as shown in AlignProp
        (https://huggingface.co/papers/2310.03739). Enabling truncated_backprop_rand allows adapting earlier
        timesteps in diffusion while not resulting in a collapse.
    gradient_checkpoint (`bool`, *optional*, defaults to True):
        Adds gradient checkpointing to Unet forward pass. Reduces GPU memory consumption while slightly increasing
        the training time.
    truncated_backprop_timestep (`int`, *optional*, defaults to 49):
        Absolute timestep to which the gradients are being backpropagated. Higher number reduces the memory usage
        and reduces the chances of collapse. While a lower value, allows more semantic changes in the diffusion
        generations, as the earlier diffusion timesteps are getting updated. However it also increases the chances
        of collapse.
    truncated_rand_backprop_minmax (`Tuple`, *optional*, defaults to (0,50)):
        Range for randomized backprop. Here the value at 0 index indicates the earlier diffusion timestep to update
        (closer to noise), while the value at index 1 indicates the later diffusion timestep to update.
    negative_prompt (`str` or `list[str]`, *optional*):
        The prompt or prompts not to guide the image generation. If not defined, one has to pass
        `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
        less than `1`).
    num_images_per_prompt (`int`, *optional*, defaults to 1):
        The number of images to generate per prompt.
    eta (`float`, *optional*, defaults to 0.0):
        Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies
        to [`schedulers.DDIMScheduler`], will be ignored for others.
    generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
        One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to
        make generation deterministic.
    latents (`torch.FloatTensor`, *optional*):
        Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
        generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
        tensor will ge generated by sampling using the supplied random `generator`.
    prompt_embeds (`torch.FloatTensor`, *optional*):
        Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
        provided, text embeddings will be generated from `prompt` input argument.
    negative_prompt_embeds (`torch.FloatTensor`, *optional*):
        Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
        If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument.
    output_type (`str`, *optional*, defaults to `"pil"`):
        The output format of the generate image. Choose between [PIL](https://pillow.readthedocs.io/en/stable/):
        `PIL.Image.Image` or `np.array`.
    return_dict (`bool`, *optional*, defaults to `True`):
        Whether to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a plain tuple.
    callback (`Callable`, *optional*):
        A function that will be called every `callback_steps` steps during inference. The function will be called
        with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
    callback_steps (`int`, *optional*, defaults to 1):
        The frequency at which the `callback` function will be called. If not specified, the callback will be
        called at every step.
    cross_attention_kwargs (`dict`, *optional*):
        A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
        `pipeline.processor` in
        [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py).
    guidance_rescale (`float`, *optional*, defaults to 0.7):
        Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
        Flawed](https://huggingface.co/papers/2305.08891) `guidance_scale` is defined as `φ` in equation 16. of
        [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891).
        Guidance rescale factor should fix overexposure when using zero terminal SNR.

Examples:

Returns:
    `DDPOPipelineOutput`: The generated image, the predicted latents used to generate the image and the associated
    log probabilities
Nr   r   r   r   r   r   r   r   F)r   use_reentrantr   r   r   r   r   Tr   r   )/r>   r   r   r   r!   no_gradr   r   r   r   rc   rf   r   r   r   rH   r   r   r   r   r   r   r   r   r   r   
checkpointrandomrandintr   r   r   r:   r   r   r   r   rB   r   r   r   r   r   r   r   r   r   )/pipeliner   r   r   r   r   r   r   r   r   r   r   r   r}   r   r   r   r   r   r   r   r   r   r   r   rp   r   r   r   r   r   r   r   r   r   r   r   r   rand_timestepr   r   r   r   r   r   r   r   s/                                                  r&   pipeline_step_with_gradr     s   h Sx}}++77(:S:SSFQX]]))558Q8QQE	"	
 *VS"9"9JJvt$<$<VJ&,,Q/J++ '5s&:# :P9["&&w5ae 	  !//!''#9. 0 	
 	(()<V(L&&00	  (}}33??**.. 	
_ 
t 9~(;>P>P>V>V(VV)KM			%8		9\i(DAq=XG9q=!9^e!)!3!3!E!EFXZ[!\ #'22MM&!+A"' 
 &]]&*7+A % +  
 " +$*NN6q9;YZ[;\%M =(%/%6%6%8
 66%/%6%6%8
 +5?5E5Ea5H2!?.?UfCf1gg
*/?#/E.z?]mn
  .h.@.@*aQX^&..G'11Hw'  * C	NQ&&AE5E+E1q5T\TfTfTlTlJlpqJq##%'A,>!,CQ7+{ ) 
:@ ("##Gll.A.A.P.P$P^c#defg"*"="=eV]M`M`"a%++a.07GH7G8h,7GH$$00K`n0oE x-..83N3N3Z##++-e[-@@_ 
z 
:	9T Is8   DS	FS7SS0S<SS-	
S
S*c                       \ rS rSrSSS.S\S\S\4S jjrS	\4S
 jrS	\4S jr	S	\
4S jr\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       rS rS rS rS rS rSrg)"DefaultDDPOStableDiffusionPipelinei!  mainT)pretrained_model_revisionuse_lorapretrained_model_namer   r   c                   [         R                  " XS9U l        X0l        Xl        X l         U R                  R                  USUS9  SU l        [        R                  " U R                  R                  R                  5      U R                  l        S U R                  l        U R                  R                  R!                  S5        U R                  R"                  R!                  S5        U R                  R$                  R!                  U R                  (       + 5        g ! [         a    U(       a  [        R                  S5         Nf = f)N)revision pytorch_lora_weights.safetensors)weight_namer  TzTrying to load LoRA weights but no LoRA weights found. Set `use_lora=False` or check that `pytorch_lora_weights.safetensors` exists in the model folder.F)r
   from_pretrainedsd_pipeliner   pretrained_modelpretrained_revisionload_lora_weightsOSErrorloggerwarningr	   from_configrH   r   safety_checkerrB   requires_grad_rK   r>   )r3   r   r   r   s       r&   __init__+DefaultDDPOStableDiffusionPipeline.__init__"  s   2BB!
 ! 5#< 	..%>2 / 
 !DM &3%>%>t?O?O?Y?Y?`?`%a"*.' 	++E2%%44U;,,->?  	U	s   "D" "&E
Er-   c                 4    [        U R                  /UQ70 UD6$ r/   )r   r  r2   s      r&   r6   +DefaultDDPOStableDiffusionPipeline.__call__A  s    T--????r%   c                 4    [        U R                  /UQ70 UD6$ r/   )r   r  r2   s      r&   rgb_with_grad0DefaultDDPOStableDiffusionPipeline.rgb_with_gradD  s    &t'7'7I$I&IIr%   c                 H    [        U R                  R                  /UQ70 UD6$ r/   )r:   r  rH   r2   s      r&   r:   1DefaultDDPOStableDiffusionPipeline.scheduler_stepG  s#    d..88J4J6JJr%   c                 .    U R                   R                  $ r/   )r  r>   r=   s    r&   r>   'DefaultDDPOStableDiffusionPipeline.unetJ  s    $$$r%   c                 .    U R                   R                  $ r/   )r  rB   r=   s    r&   rB   &DefaultDDPOStableDiffusionPipeline.vaeN  s    ###r%   c                 .    U R                   R                  $ r/   )r  rE   r=   s    r&   rE   ,DefaultDDPOStableDiffusionPipeline.tokenizerR      )))r%   c                 .    U R                   R                  $ r/   )r  rH   r=   s    r&   rH   ,DefaultDDPOStableDiffusionPipeline.schedulerV  r  r%   c                 .    U R                   R                  $ r/   )r  rK   r=   s    r&   rK   /DefaultDDPOStableDiffusionPipeline.text_encoderZ  s    ,,,r%   c                 H    U R                   (       a  [        R                  $ S $ r/   )r   
contextlibnullcontextr=   s    r&   rN   +DefaultDDPOStableDiffusionPipeline.autocast^  s    )-z%%@D@r%   c                     U R                   (       aA  [        [        U R                  R                  5      5      nU R                  R                  XS9  U R                  R                  U5        g )Nsave_directoryunet_lora_layers)r   r   r   r  r>   save_lora_weightsrU   )r3   
output_dir
state_dicts      r&   rU   2DefaultDDPOStableDiffusionPipeline.save_pretrainedb  sQ    ==89RSWScScShSh9ijJ..j.f((4r%   c                 <    U R                   R                  " U0 UD6  g r/   )r  rQ   r2   s      r&   rQ   :DefaultDDPOStableDiffusionPipeline.set_progress_bar_configh  s    00$A&Ar%   c                    U R                   (       a  [        SSS/ SQS9nU R                  R                  R	                  U5        U R                  R                  R                  5        H:  nUR                  (       d  M  UR                  [        R                  5      Ul
        M<     U R                  R                  $ U R                  R                  $ )N   gaussian)to_kto_qto_vzto_out.0)r
lora_alphainit_lora_weightstarget_modules)r   r   r  r>   add_adapter
parametersrequires_gradro   r!   float32data)r3   lora_configparams      r&   rX   7DefaultDDPOStableDiffusionPipeline.get_trainable_layersk  s    ==$",C	K !!--k: ))..99;&&&!&%--!8EJ < ##(((##(((r%   c                    [        U5      S:w  a  [        S5      eU R                  (       aV  [        US   S5      (       aB  [	        US   SS 5      b1  [        [        US   5      5      nU R                  R                  X4S9  g U R                  (       dK  [        US   [        5      (       a3  US   R                  [        R                  R                  US5      5        g [        S[        US   5       35      e)Nr   CGiven how the trainable params were set, this should be of length 1r   peft_configr)  r>   Unknown model type )rc   rd   r   r   getattrr   r   r  r,  r   r   rU   ospathjoinr   )r3   modelsweightsr-  r.  s        r&   r[   2DefaultDDPOStableDiffusionPipeline.save_checkpoint~  s    v;!bcc==WVAY>>76RS9VceiCjCv89RSYZ[S\9]^J..j.f:fQi9M#N#N1I%%bggll:v&FG24q	?2CDEEr%   c                    [        U5      S:w  a  [        S5      eU R                  (       a:  U R                  R	                  USS9u  p4U R                  R                  X4US   S9  g U R                  (       dp  [        US   [        5      (       aX  [        R                  " USS9nUS   R                  " S
0 UR                  D6  US   R                  UR                  5       5        Ag [        S	[        US   5       35      e)Nr   rE  r  )r  r   )network_alphasr>   r>   )	subfolderrG  r   )rc   rd   r   r  lora_state_dictload_lora_into_unetr   r   r  register_to_configr   load_state_dictr.  r   )r3   rL  	input_dirrR  rP  
load_models         r&   r^   2DefaultDDPOStableDiffusionPipeline.load_checkpoint  s    v;!bcc==.2.>.>.N.N'I /O /+O 00flmnfo0p:fQi9M#N#N-==iSYZJ1I((=:+<+<=1I%%j&;&;&=>24q	?2CDEEr%   )r  r  r  r   N)r   r   r   r   r   boolr  r   r6   r  r(   r:   r`   r>   rB   rE   rH   rK   rN   rU   rQ   rX   r[   r^   r$   r   r%   r&   r   r   !  s    W]pt @c @QT @im @>@+= @J0B JK1D K % % $ $ * * * * - - A A5B)&	FFr%   r   )r   FNN)NNN2         @Nr   r   NNNNpilTNr   Nr   )NNNrZ  r[  TTT1   )r   rZ  Nr   r   NNNNr\  TNr   Nr   )8r%  rI  r   dataclassesr   typingr   r   r   r   numpyr   r!   torch.utils.checkpointutilsr   
accelerater   	diffusersr	   r
   r   >diffusers.pipelines.stable_diffusion.pipeline_stable_diffusionr   transformers.utilsr   corer   sd_utilsr   peftr   
peft.utilsr   
get_loggerr   r  r   r(   r+   rj   rz   FloatTensorintfloatrY  r:   r   r   r   	Generatordictr   r   r   r   r   r%   r&   <module>rq     s    	  ! 1 1   + +  R R \ 0  5 4 
		H	%   &   Q" Q"hk ( %*/3BI##BI BI 	BI
 
BI #BI %++,BI BIN  /3 !7;+,IM+/15:>!&HL7;!'MAU3S	>*+MA SMMA C=	MA
 MA MA eCcN34MA $C=MA 
MA eootEOO/DDEFMA e''(MA E--.MA %U%6%67MA #MA MA  xc5+<+< =t CDE!MA" #MA$ %T#s(^4%MA& 'MA MAd /3 !#$( $'),37;+,IM+/15:>!&HL7;!1FAU3S	>*+FA SMFA C=	FA
 FA FA FA "FA FA "%FA %*FA eCcN34FA $C=FA 
FA eootEOO/DDEFFA  e''(!FA" E--.#FA$ %U%6%67%FA& #'FA( )FA* xc5+<+< =t CDE+FA, -FA. %T#s(^4/FA0 1FARwF)D wFr%   