
    +hA?             +          S r SSKrSSKJr  SSKJr  SSKJrJrJ	r	  SSK
r
SSKrSSKrSSKJrJrJrJrJrJrJrJrJr  SSKJrJrJrJr  SS	KJrJrJrJ r J!r!J"r"J#r#J$r$J%r%  SS
K&J'r'J(r(  SSK)J*r*  SSK+J,r,  SSK-J.r.J/r/  SSK0J1r1  SSK2J3r3  SSK4J5r5  SSK6J7r7  \'" 5       (       a  SSK8J9r9  SSK:J;r;  \(Rx                  " \=5      r>SGS jr?SHS jr@SHS jrASHS jrBSHS jrC SIS jrDS rESJS\F4S jjrGS\F4S jrHS  rIS! rJ SKS" jrKS# rLS$ rMSLS% jrN/ S&QrO\O V s0 s H  o S   U S   _M     sn rP/ S'QrQ\Q V s0 s H  n \R                  U S   5      U S   _M     sn rS\R                  " S(R                  \SR                  5       5      5      rWSJS) jrX   SMS* jrYSJS+ jrZ SNS,\\[   S-\\[   4S. jjr\  SNS/ jr]                           SOS0\	\[\\[\R                  4   4   S1\[S\\F   S2\[S3\[S4\_S5\[S6\\F   S7\\_   S-\[S8\_S9\\[   S:\\[   S,\\[   S;\\_   S<\\_   S=\_S>\\5   S?\\   S@\3SA\34*SB jjr`        SPSC\[S1\[S\FS4\_S6\\F   S7\\_   S-\[S8\_SD\\_   SE\\_   SA\34SF jjrags  sn f s  sn f )Qz7Conversion script for the Stable Diffusion checkpoints.    N)nullcontext)BytesIO)DictOptionalUnion)	AutoFeatureExtractorBertTokenizerFastCLIPImageProcessorCLIPTextConfigCLIPTextModelCLIPTextModelWithProjectionCLIPTokenizerCLIPVisionConfigCLIPVisionModelWithProjection   )AutoencoderKLControlNetModelPriorTransformerUNet2DConditionModel)	DDIMSchedulerDDPMSchedulerDPMSolverMultistepSchedulerEulerAncestralDiscreteSchedulerEulerDiscreteSchedulerHeunDiscreteSchedulerLMSDiscreteSchedulerPNDMSchedulerUnCLIPScheduler)is_accelerate_availablelogging)DIFFUSERS_REQUEST_TIMEOUT)
get_device   )LDMBertConfigLDMBertModel)PaintByExampleImageEncoder)DiffusionPipeline   )StableDiffusionSafetyChecker)StableUnCLIPImageNormalizer)init_empty_weights)set_module_tensor_to_devicec                     US:  a#  SR                  U R                  S5      US 5      $ SR                  U R                  S5      SU 5      $ )z_
Removes segments. Positive values shave the first segments, negative shave the last segments.
r   .N)joinsplit)pathn_shave_prefix_segmentss     p/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/pipelines/stable_diffusion/convert_from_ckpt.pyshave_segmentsr4   G   sL     !#xx

3(?(@ABBxx

3(@)@ABB    c                 ,   / nU  H  nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  S	S
5      nUR                  SS5      n[        XAS9nUR                  X4S.5        M     U$ )H
Updates paths inside resnets to the new naming scheme (local renaming)
zin_layers.0norm1zin_layers.2conv1zout_layers.0norm2zout_layers.3conv2zemb_layers.1time_emb_projskip_connectionconv_shortcutr2   oldnewreplacer4   appendold_listr2   mappingold_itemnew_items        r3   renew_resnet_pathsrK   Q   s     G##M7;##M7;##NG<##NG<##NOD##$5G!(\x9:  Nr5   c                 |    / nU  H3  nUnUR                  SS5      n[        XAS9nUR                  X4S.5        M5     U$ )r7   nin_shortcutr>   r?   r@   rC   rF   s        r3   renew_vae_resnet_pathsrN   g   sJ     G##NOD!(\x9:  Nr5   c                 F    / nU  H  nUnUR                  X4S.5        M     U$ )K
Updates paths inside attentions to the new naming scheme (local renaming)
r@   )rE   rF   s        r3   renew_attention_pathsrQ   w   s0     G 	x9:  Nr5   c                    / nU  H  nUnUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  S	S
5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      n[        XAS9nUR                  X4S.5        M     U$ )rP   znorm.weightzgroup_norm.weightz	norm.biaszgroup_norm.biaszq.weightzto_q.weightzq.biasz	to_q.biaszk.weightzto_k.weightzk.biasz	to_k.biaszv.weightzto_v.weightzv.biasz	to_v.biasproj_out.weightzto_out.0.weightproj_out.biaszto_out.0.biasr?   r@   rC   rF   s        r3   renew_vae_attention_pathsrU      s     G##M3FG##K1BC##J>##Hk:##J>##Hk:##J>##Hk:##$57HI##O_E!(\x9:) , Nr5   c                    [        U [        5      (       d   S5       eUb  UR                  5        H  u  pgX&   nUR                  S   S-  n	[	        UR                  5      S:X  a  SU	4OSn
UR                  S   US   -  S-  nUR                  USU	-  U-  4UR                  SS -   5      nUR                  X-  SS9u  pnUR                  U
5      XS	   '   UR                  U
5      XS
   '   UR                  U
5      XS   '   M     U  H  nUS   nUb  X;   a  M  UR                  SS5      nUR                  SS5      nUR                  SS5      nUb!  U H  nUR                  US   US   5      nM     SU;   =(       d    SU;   =(       a    SU;   nX&S      R                  nU(       a$  [	        U5      S:X  a  X&S      SS2SS2S4   X'   M  U(       a%  [	        U5      S:X  a  X&S      SS2SS2SS4   X'   M  X&S      X'   M     g)z
This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits
attention layers, and takes into account additional replacements that may arise.

Assigns the weights to the new checkpoint.
z@Paths should be a list of dicts containing 'old' and 'new' keys.Nr   r   num_head_channelsr(   )dimquerykeyvaluerB   zmiddle_block.0zmid_block.resnets.0middle_block.1mid_block.attentions.0zmiddle_block.2zmid_block.resnets.1rA   proj_attn.weight
attentionsto_   )
isinstancelistitemsshapelenreshaper0   rD   )paths
checkpointold_checkpointattention_paths_to_splitadditional_replacementsconfigr1   path_map
old_tensorchannelstarget_shape	num_headsrZ   r[   r\   new_pathreplacementis_attn_weightrf   s                      r3   assign_to_checkpointrw      sG    eT""f$ff"  +6<<>ND'-J!''*a/H-01A1A-Ba-GB>bL"((+v6I/JJaOI#++YH	8Q,RU_UeUefgfhUi,ijJ * 0 01FA 0 NE,1MM,,GJ()*-++l*CJ',1MM,,GJ() ? ; $/H4X ##$46KL##$46NO##$46KL".6#++K,>E@RS  7 ,x7kLH<T<jY^bjYjE{+11c%jAo#1u+#>q!Qw#GJ E
a#1u+#>q!Qz#JJ #1u+#>J 1 r5   c                 D   [        U R                  5       5      n/ SQnU H}  nSR                  UR                  S5      SS  5      U;   a'  X   R                  S:  a  X   S S 2S S 2SS4   X'   MO  MQ  SU;   d  MY  X   R                  S:  d  Mm  X   S S 2S S 2S4   X'   M     g )N)zquery.weightz
key.weightzvalue.weightr.   r#   r   r_   )rd   keysr/   r0   ndim)rj   rz   	attn_keysr[   s       r3   conv_attn_to_linearr}      s    
!"D>I88CIIcN23'(I5##a'",/!Q1*"=
 (3&##a'",/!Q'":
 r5   
image_sizec                 $   U(       a  U S   S   S   S   nO5SU S   S   ;   a  U S   S   S   b  U S   S   S   S   nOU S   S   S   S   nU S   S   S   S   S   nUS	    Vs/ s H
  oSS
   U-  PM     nn/ nSn[        [        U5      5       H9  n	XS   ;   a  SOSn
UR                  U
5        U	[        U5      S-
  :w  d  M4  US-  nM;     / n[        [        U5      5       H%  n	XS   ;   a  SOSn
UR                  U
5        US-  nM'     US   b,  [        US   [        5      (       a  US   O[        US   5      nOSnS[        US   5      S-
  -  nSU;   a  US   OSnSU;   a  US   OSnU(       a/  Uc,  US
   US   -  n[        US	   5       Vs/ s H  nUU-  PM
     nnSnSnSnSnSnUS   b%  [        US   [        5      (       a  US   OUS   S   nSU;   a#  US   S:X  a  US;   a  SnSnOSnS U;   d   eUS    nX-  US!   [        U5      [        U5      US"   UUUUUUUUS#.nS$U;   a  US$   US%'   SU;   a   [        US   [        5      (       a  US   US&'   U(       a
  US'   US('   U$ US)   US)'   [        U5      US*'   U$ s  snf s  snf )+J
Creates a config for the diffusers based on the config of the LDM model.
modelparamscontrol_stage_configunet_configNnetwork_configfirst_stage_configddconfigchannel_multmodel_channelsr(   attention_resolutionsCrossAttnDownBlock2DDownBlock2Dr#   CrossAttnUpBlock2D	UpBlock2Dtransformer_depthch_multrs   use_linear_in_transformerFrX   context_dimr   num_classes
sequential)      	text_time   
projectionadm_in_channelsin_channelsnum_res_blocks)sample_sizer   down_block_typesblock_out_channelslayers_per_blockcross_attention_dimattention_head_dimuse_linear_projectionclass_embed_typeaddition_embed_typeaddition_time_embed_dim%projection_class_embeddings_input_dimtransformer_layers_per_blockdisable_self_attentionsonly_cross_attentionnum_class_embedshint_channelsconditioning_channelsout_channelsup_block_types)rangerg   rE   rc   intrd   tuple)original_configr~   
controlnetunet_params
vae_paramsmultr   r   
resolutioni
block_typer   r   vae_scale_factorhead_dimr   head_dim_multcr   r   r   r   r   rn   s                           r3   create_unet_diffusers_configr      s    %g.x89OPQYZ _W5h??(2=AM)'28<]KHUK)'28<=MNxXK )(34HI(ST^_JKVWeKfgKf4&67$>KfgJ3)*+/9I`=a/a+gt

+&'!++!OJ	 , N3)*+-7G^;_-_)ep
j)q
 ,
 &'3 +&9:C@@ +,k"567 	% ()$SI!67!;<+6++E{;'4H4OS^4^/0di  '(89[I\=]]M37N8S3TU3Ta)3THU",0)K=!- +m4c:: &]+A. 	 #}%5l*&1#*-'#/ $3334?@Q4R1 "5"=1!"23#$67'(89*&!6,2#:1V(DF  !K/)45N)O%&#
;}3Ms(S(S%0%?!"*5o*F&'
 M "-^!<~#(#8 Ms hF Vs   $JJc           	         U S   S   S   S   S   nU S   S   S   S   S   nUS    Vs/ s H
  oBS   U-  PM     nn[        U5       VVs/ s H  u  pcUSU-  -  US	   ;  a  S
OSPM     nnn[        U5       VVs/ s H  u  pcUSU-  -  US	   ;  a  SOSPM     snnSSS2   nUUS   US   [        U5      [        U5      [        U5      US   US   S.n	U	$ s  snf s  snnf s  snnf )r   r   r   r   r   	embed_dimr   chr#   attn_resolutionsDownEncoderBlock2DAttnDownEncoderBlock2DUpDecoderBlock2DAttnUpDecoderBlock2DNrW   r   out_ch
z_channelsr   )r   r   r   r   r   r   latent_channelsr   )	enumerater   )
r   r~   r   _r   r   r   r   r   rn   s
             r3   create_vae_diffusers_configr   Z  s^    !)(34HI(ST^_J *+?@J;WA>H>ST>SdT*T1>ST 011DA !+ad 2*EW:X X^vv1   011DA )AqD0
CU8VV\rr1 dN "!-0"8,!"23/#$67%l3&'78	F M) Us   CC;C#c                 T    [        U S   S   S   U S   S   S   U S   S   S   SS9nU$ )Nr   r   	timestepslinear_start
linear_endscaled_linear)num_train_timesteps
beta_startbeta_endbeta_schedule)r   )r   	schedulars     r3   create_diffusers_schedularr   x  sM    +G4X>{K"7+H5nE )(3LA%	I r5   c                 z    U S   S   S   S   n[        UR                  UR                  UR                  S-  S9nU$ )Nr   r   cond_stage_configrb   )d_modelencoder_layersencoder_ffn_dim)r$   n_embedn_layer)r   bert_paramsrn   s      r3   create_ldm_bert_configr     sM    !'*845HI(SK##"**#++a/F
 Mr5   c           
      D   U(       a  U nGOL0 n[        U R                  5       5      nU(       a  SnOSn[        S U 5       5      S:  a  U(       a  [        R	                  SU S35        [        R	                  S5        U Hd  n	U	R                  S5      (       d  M  S	S
R                  U	R                  S5      SS 5      -   n
U R                  U
5      XiR                  US
5      '   Mf     Oo[        S U 5       5      S:  a  [        R	                  S5        U H>  n	U	R                  U5      (       d  M  U R                  U	5      XiR                  US
5      '   M@     0 nUS   US'   US   US'   US   US'   US   US'   US   c  ODUS   S:X  d	  US   S:X  a!  US   US'   US   US'   US   US '   US!   US"'   O[        S#US    35      eUS$   S%:X  a   US   US&'   US   US''   US   US('   US!   US)'   S*U;   a  US*   b  S+U;   a  US+   US,'   US-   US.'   US/   US0'   U(       d   US1   US2'   US3   US4'   US5   US6'   US7   US8'   [        U Vs1 s H.  nS9U;   d  M  SR                  UR                  S5      SS: 5      iM0     sn5      n[        U5       VV	s0 s H!  nX V	s/ s H  n	S;U 3U	;   d  M  U	PM     sn	_M#     nnn	[        U Vs1 s H.  nS<U;   d  M  SR                  UR                  S5      SS: 5      iM0     sn5      n[        U5       VV	s0 s H!  nX V	s/ s H  n	S=U 3U	;   d  M  U	PM     sn	_M#     nnn	[        U Vs1 s H.  nS>U;   d  M  SR                  UR                  S5      SS: 5      iM0     sn5      n[        U5       VV	s0 s H!  nX V	s/ s H  n	S?U 3U	;   d  M  U	PM     sn	_M#     nnn	[        SU5       GH  nUS-
  US@   S-   -  nUS-
  US@   S-   -  nUU    V	s/ s H  n	S;U SA3U	;   d  M  S;U SB3U	;  d  M  U	PM     nn	UU    V	s/ s H  n	S;U SC3U	;   d  M  U	PM     nn	S;U SD3U;   a8  UR                  S;U SD35      USEU SF3'   UR                  S;U SG35      USEU SH3'   [        U5      nS;U SA3SEU SIU 3SJ.n[        UXU/USK9  [        U5      (       d  M  [        U5      nS;U SC3SEU SLU 3SJ.n[        UXU/USK9  GM     USM   nUS   nUS:   n[        U5      n[        UXUSN9  [        U5      n[        UXUSN9  [        U5      nSOSPSJ.n[        UXU/USK9  [        U5       GH\  nUUS@   S-   -  nUUS@   S-   -  nUU    V s/ s H  n [!        U S:5      PM     n!n 0 n"U! HE  nUR                  S5      SM   [!        US5      n#nUU";   a  U"U   R#                  U#5        M?  U#/U"U'   MG     [        U"5      S:  Ga`  UU    V	s/ s H  n	S?U SA3U	;   d  M  U	PM     nn	UU    V	s/ s H  n	S?U SC3U	;   d  M  U	PM     nn	[        U5      n[        U5      nS?U SA3SQU SIU 3SJ.n[        UXU/USK9  [%        U"R'                  5       5       V$V%s0 s H  u  n$n%U$[%        U%5      _M     n"n$n%SRSS/U"R)                  5       ;   aa  [        U"R)                  5       5      R+                  SRSS/5      n&US?U SU& ST3   USQU SU3'   US?U SU& SV3   USQU SW3'   [        U5      S::X  a  / n[        U5      (       a*  [        U5      nS?U SC3SQU SLU 3SJ.n[        UXU/USK9  GM  GM  [        U!SSX9nU HV  nSR                  S>[-        U5      USY   /5      n'SR                  SZ[-        U5      S[[-        U5      US\   /5      n(UU'   UU('   MX     GM_     U(       Ga+  SMn)UR                  S]U) S^35      US_'   UR                  S]U) S`35      USa'   U)S:-  n)SMn*U*Sb:  aJ  UR                  S]U) S^35      UScU* S^3'   UR                  S]U) S`35      UScU* S`3'   U*S-  n*U)S:-  n)U*Sb:  a  MJ  UR                  S]U) S^35      USd'   UR                  S]U) S`35      USe'   [        U5       H;  nUR                  SfU Sg35      UShU S^3'   UR                  SfU Si35      UShU S`3'   M=     UR                  Sj5      USk'   UR                  Sl5      USm'   U$ s  snf s  sn	f s  sn	nf s  snf s  sn	f s  sn	nf s  snf s  sn	f s  sn	nf s  sn	f s  sn	f s  sn f s  sn	f s  sn	f s  sn%n$f )nzF
Takes a state dict and a config, and returns a converted checkpoint.
zcontrol_model.zmodel.diffusion_model.c              3   B   #    U  H  oR                  S 5      v   M     g7f	model_emaN
startswith.0ks     r3   	<genexpr>.convert_ldm_unet_checkpoint.<locals>.<genexpr>  s     7$Q||K(($   d   zCheckpoint z" has both EMA and non-EMA weights.zIn this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag.zmodel.diffusion_modelz
model_ema. r.   r(   Nc              3   B   #    U  H  oR                  S 5      v   M     g7fr   r   r   s     r3   r   r     s     ;d<<,,dr   zIn this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA weights (usually better for inference), please make sure to add the `--extract_ema` flag.time_embed.0.weightztime_embedding.linear_1.weightztime_embed.0.biasztime_embedding.linear_1.biasztime_embed.2.weightztime_embedding.linear_2.weightztime_embed.2.biasztime_embedding.linear_2.biasr   timestepr   zlabel_emb.0.0.weightzclass_embedding.linear_1.weightzlabel_emb.0.0.biaszclass_embedding.linear_1.biaszlabel_emb.0.2.weightzclass_embedding.linear_2.weightzlabel_emb.0.2.biaszclass_embedding.linear_2.biasz$Not implemented `class_embed_type`: r   r   zadd_embedding.linear_1.weightzadd_embedding.linear_1.biaszadd_embedding.linear_2.weightzadd_embedding.linear_2.biasr   zlabel_emb.weightzclass_embedding.weightzinput_blocks.0.0.weightzconv_in.weightzinput_blocks.0.0.biaszconv_in.biaszout.0.weightzconv_norm_out.weightz
out.0.biaszconv_norm_out.biaszout.2.weightzconv_out.weightz
out.2.biaszconv_out.biasinput_blocksr#   zinput_blocks.middle_blockzmiddle_block.output_blockszoutput_blocks.r   z.0z.0.opz.1z.0.op.weightdown_blocks..downsamplers.0.conv.weightz
.0.op.bias.downsamplers.0.conv.biasz	.resnets.r@   rm   rn   z.attentions.r   )rn   r]   r^   
up_blocks.z	conv.biaszconv.weightz.conv.weight.upsamplers.0.conv.weightz
.conv.bias.upsamplers.0.conv.biasr?   rA   	up_blocksresnetsrB   zinput_hint_block.z.weightz(controlnet_cond_embedding.conv_in.weightz.biasz&controlnet_cond_embedding.conv_in.bias   z!controlnet_cond_embedding.blocks.z)controlnet_cond_embedding.conv_out.weightz'controlnet_cond_embedding.conv_out.biaszzero_convs.z	.0.weightzcontrolnet_down_blocks.z.0.biaszmiddle_block_out.0.weightzcontrolnet_mid_block.weightzmiddle_block_out.0.biaszcontrolnet_mid_block.bias)rd   rz   sumloggerwarningr   r/   r0   poprD   NotImplementedErrorrg   r   rK   rw   rQ   r4   rE   sortedre   valuesindexstr)+rj   rn   r1   extract_emar   skip_extract_state_dictunet_state_dictrz   unet_keyr[   flat_ema_keynew_checkpointlayernum_input_blockslayer_idr   num_middle_blocksmiddle_blocksnum_output_blocksr   r   block_idlayer_in_block_idr   r`   ri   	meta_pathresnet_0resnet_1resnet_0_pathsresnet_1_pathsattentions_pathsnameoutput_block_layersoutput_block_list
layer_namer   vr  old_pathrt   
orig_indexdiffusers_indexs+                                              r3   convert_ldm_unet_checkpointr%    s    $ JOO%&'H/H 7$77#=+NN[.PQRNNr >>"9::#/"''#))C.:L2M#MLAKP\A]OKK"$=> 
 ;d;;cAq
 >>(++AKPSATOKK"$=>  N7FG\7]N345DEX5YN127FG\7]N345DEX5YN12 !)	"	#z	1V<N5OS_5_<KLb<c89:IJ^:_67<KLb<c89:IJ^:_67!$HPbIcHd"eff#$3:IJ`:a678GH\8]45:IJ`:a678GH\8]45 V#%&29K9^7FGY7ZN34'67P'QN#$%45L%MN>"1@1P-./>|/L+,,;N,K()*9,*G' su[imr[r:CHHU[[%5bq%9:st .//H 	/W/3}XJ5OSV5V3/WW/   t\jns\s;SXXekk#&6r&:;tu /00H 	/W/3}XJ5OSV5V3/WW0   u\kot\t;SXXekk#&6r&:;uv /00H 	/X/3~hZ5PTW5W3/XX0  
 1&'Ev&89A=>Uv.@'AA'EF (?
*CaS.Cs.JCQ^_`^aafOgorOrC? 	 
 &2!_U_c-s"8MQT8Tc_
U1#\*o=SbSfSfs,/TN\(3NOP RaQdQds*-RN\(3LMN #7++A3b1L
R[\m[n:op	>YK`f	
 z??)*5E"/s" 5XJVbctbu>vwI ~QZP[dj7 (> QHq!JQH'1NQWX'1NQWX,Z8(1IJI.T]S^gm $% 23a78(:!;a!?@CPQRCSTCS4~dA6CST(E#(;;s#3A#6ua8PjH,,!(+22:>/9l!(+ )  !A%&3A&6X&6sN1#R:PTW:Ws&6GX)6q)9[)9#~aSPR=SWZ=Z#)9J[/8N&w/E"02 6H:U^_p^q?rsI ~QZP[dj ;AARAXAXAZ:[ \:[$!QF1I:[ \]+/@/G/G/II.5578>>]?[\Sb$QCq|<TH:5NOP Ra$QCqz:RH:5LMN
 z?a'!#J:-j9+A3b1'z>O=PQ	 %>U^T_hn  00C]^_N&88_c!fd5k$JK88[#h-CPaLbdhindo$pq+:8+Dx(	 'e &p  
ETEXEX
|73F
AB DSCVCV
|51D
?@ 	a
![j[n[n#J<w7\N>>OwWX ZiYlYl#J<u5ZN>>OuUV q O!OJ ! GVFYFY
|73G
BC ETDWDW
|51E
@A
 '(ACRCVCVYdefdggpWqCrN4QCw?@APATATWbcdbeelUmAnN4QCu=> )
 9H8K8KLg8h456E6I6IJc6d23K tW uW vX
 VV U Y[ !]s   
c'c
cc-c3c
c#'c#
c-c(+c(1c-
c3'c3
c=c8)c8/c=,d>d
dd+ddd,d;dd#dc(c-8c=c           
      	   0 n[        U R                  5       5      n[        S U 5       5      (       a  SOSnU H>  nUR                  U5      (       d  M  U R	                  U5      X%R                  US5      '   M@     0 nUS   US'   US   US'   US   US'   US   US'   US   US	'   US
   US'   US   US'   US   US'   US   US'   US   US'   US   US'   US   US'   US   US'   US   US'   US   US'   US   US'   [        U Vs1 s H.  nSU;   d  M  SR                  UR                  S5      S S 5      iM0     sn5      n[        U5       V	Vs0 s H!  oU Vs/ s H  nSU	 3U;   d  M  UPM     sn_M#     n
n	n[        U Vs1 s H.  nSU;   d  M  SR                  UR                  S5      S S 5      iM0     sn5      n[        U5       V	Vs0 s H!  oU Vs/ s H  nSU	 3U;   d  M  UPM     sn_M#     nn	n[        U5       H  nX    Vs/ s H  nSU 3U;   d  M  SU S3U;  d  M  UPM     nnSU S 3U;   a8  UR                  SU S 35      US!U S"3'   UR                  SU S#35      US!U S$3'   [        U5      nSU S%3S&U S'3S(.n[        XUU/US)9  M     U Vs/ s H  nS*U;   d  M  UPM     nnS+n[        S,US,-   5       HF  nU Vs/ s H  nS-U 3U;   d  M  UPM     nn[        U5      nS.U 3S/US,-
   3S(.n[        XUU/US)9  MH     U Vs/ s H  nS0U;   d  M  UPM     nn[        U5      nS1S2S(.n[        XUU/US)9  [        U5        [        U5       H  nUS,-
  U-
  nUU    Vs/ s H  nSU 3U;   d  M  SU S33U;  d  M  UPM     nnS4U S53U;   a   US4U S53   US6U S73'   US4U S83   US6U S93'   [        U5      nSU S%3S:U S'3S(.n[        XUU/US)9  M     U Vs/ s H  nS;U;   d  M  UPM     nnS+n[        S,US,-   5       HF  nU Vs/ s H  nS<U 3U;   d  M  UPM     nn[        U5      nS.U 3S/US,-
   3S(.n[        XUU/US)9  MH     U Vs/ s H  nS=U;   d  M  UPM     nn[        U5      nS1S2S(.n[        XUU/US)9  [        U5        U$ s  snf s  snf s  snn	f s  snf s  snf s  snn	f s  snf s  snf s  snf s  snf s  snf s  snf s  snf s  snf )>Nc              3   B   #    U  H  oR                  S 5      v   M     g7f)first_stage_model.Nr   r   s     r3   r   -convert_ldm_vae_checkpoint.<locals>.<genexpr>  s     )[VZQR,,7K*L*LVZr   r(  r   zencoder.conv_in.weightzencoder.conv_in.biaszencoder.conv_out.weightzencoder.conv_out.biaszencoder.norm_out.weightzencoder.conv_norm_out.weightzencoder.norm_out.biaszencoder.conv_norm_out.biaszdecoder.conv_in.weightzdecoder.conv_in.biaszdecoder.conv_out.weightzdecoder.conv_out.biaszdecoder.norm_out.weightzdecoder.conv_norm_out.weightzdecoder.norm_out.biaszdecoder.conv_norm_out.biaszquant_conv.weightzquant_conv.biaszpost_quant_conv.weightzpost_quant_conv.biaszencoder.downr.   r   zdown.z
decoder.upzup.z.downsamplezencoder.down.z.downsample.conv.weightzencoder.down_blocks.r   z.downsample.conv.biasr   z.blockr   z.resnetsr@   r   zencoder.mid.blockr#   r(   zencoder.mid.block_z
mid.block_zmid_block.resnets.zencoder.mid.attnz
mid.attn_1r^   z	.upsamplezdecoder.up.z.upsample.conv.weightzdecoder.up_blocks.r   z.upsample.conv.biasr   r   zdecoder.mid.blockzdecoder.mid.block_zdecoder.mid.attn)rd   rz   anyr   getrD   rg   r/   r0   r   r  rN   rw   rU   r}   )rj   rn   vae_state_dictrz   vae_keyr[   r  r  num_down_blocksr  down_blocksnum_up_blocksr   r   r   ri   r  mid_resnetsnum_mid_res_blocksmid_attentionsr  s                        r3   convert_ldm_vae_checkpointr4    s   N
!"D&))[VZ)[&[&["acG>>'""7A~~c7JN;;w34  N/=>V/WN+,-;<R-SN)*0>?X0YN,-.<=T.UN*+5CD]5^N123ABY3ZN/0/=>V/WN+,-;<R-SN)*0>?X0YN,-.<=T.UN*+5CD]5^N123ABY3ZN/0*89L*MN&'(67H(IN$%/=>V/WN+,-;<R-SN)* ~q~eYgkpYp9388EKK$4Ra$89~qrO_det_u_uS[.N.3eH:4F#4M3.NN_u  
 ^m^EWcglWl7%++c"22A"67^mnM]bcp]q]qQY.L.3c(4D4K3.LL]q   ?#"-.m.3eA3K34F3USTRUU`KailKl3.m1#45GTbTfTfs"9:UN1!4OPQ SaRdRds"78SN1!4MNO 'w/#A3f-QCx6PQ	UN]f\gpvw $ #1O.34G34N3.KO1(1,-"-Q+33EaS1IS1P3+Q&w/(,7I!a%5QR	UN]f\gpvw . &4Q^c7IS7Pc^NQ%n5E$-EFIYbXclrs'=! 1$q($X.
.CCz2Bc2ICPST\S]]fNgorNrC. 	 
 
"78NJP^hZ'<=QN/s2KLM O]hZ':;ON/s2IJK 'w/!(62Zs(;ST	UN]f\gpvw! "$ #1O.34G34N3.KO1(1,-"-Q+33EaS1IS1P3+Q&w/(,7I!a%5QR	UN]f\gpvw . &4Q^c7IS7Pc^NQ%n5E$-EFIYbXclrs'S rN
 nL
 n P R R
  P R Rs   
R"'R"
R,R'+R'1R,
R2'R2
R<R7)R7/R<SS*S!
S/SS S
S$S1SSS-
S;SS ,S "
S%0S%'R,7R<c                   ^^^ S mS mUU4S jmU4S jn[        U5      R                  5       nU R                  R                  R                  UR
                  R                  l        U R                  R                  R                  R                  UR
                  R                  R                  l
        T" UR
                  R                  U R                  R                  5        U" UR
                  R                  U R                  R                  R                  5        T" UR                  U R                  R                  5        U$ )Nc                    UR                   R                  U R                  R                  l        UR                  R                  U R
                  R                  l        UR                  R                  U R                  R                  l        UR                  R                  U R                  l        UR                  R                  U R                  l
        g N)to_qweightq_projdatato_kk_projto_vv_projto_outout_projbias)hf_attn_layerpt_attn_layers     r3   _copy_attn_layer5convert_ldm_bert_checkpoint.<locals>._copy_attn_layer  s    +8+=+=+D+D##(+8+=+=+D+D##(+8+=+=+D+D##((5(<(<(C(C%&3&:&:&?&?#r5   c                 H    UR                   U l         UR                  U l        g r7  )r9  rB  )	hf_linear	pt_linears     r3   _copy_linear1convert_ldm_bert_checkpoint.<locals>._copy_linear  s    $++	"	r5   c                 2  > T" U R                   US   S   5        T" U R                  US   S   5        T" U R                  US   S   5        US   S   nT" U R                  UR                  S   S   5        T" U R
                  UR                  S   5        g )Nr   r(   r#   )self_attn_layer_normfinal_layer_norm	self_attnfc1netfc2)hf_layerpt_layerpt_mlprE  rJ  s      r3   _copy_layer0convert_ldm_bert_checkpoint.<locals>._copy_layer  s    X22HQKNCX..A? 	++Xa[^< !QX\\6::a=#34X\\6::a=1r5   c                 `   > [        U 5       H  u  p#US:w  a  X"-  nXUS-    nT" X45        M      g )Nr   r#   )r   )	hf_layers	pt_layersr   rS  rT  rV  s        r3   _copy_layers1convert_ldm_bert_checkpoint.<locals>._copy_layers	  s8    $Y/KAAv QU+H+	 0r5   )r%   evaltransformer	token_embr9  r   embed_tokenspos_embembembed_positionsr;  
layer_normnormlayersattn_layers	to_logits)rj   rn   r[  hf_modelrE  rV  rJ  s       @@@r3   convert_ldm_bert_checkpointrj    s    @(2, F#((*H *4)?)?)I)I)P)PHNN&1;1G1G1O1O1S1S1Z1ZHNN"")). **J,B,B,G,GH &&
(>(>(J(J(Q(QR##Z%;%;%E%EFOr5   c           	         UcN  Sn [         R                  " X1S9n[	        5       (       a  [
        O[        nU" 5          [        U5      nS S S 5        OUn[        U R                  5       5      n0 nSS/n	U H9  n
U	 H0  nU
R                  U5      (       d  M  X
   X[        US-   5      S  '   M2     M;     [	        5       (       a'  UR                  5        H  u  p[        WUSUS	9  M     W$ [        WS
5      (       a$  [        UR                  R                   5      (       d  UR#                  SS 5        UR%                  U5        U$ ! [         a    [        SU S35      ef = f! , (       d  f       GN= f)Nopenai/clip-vit-large-patch14local_files_onlyWith local_files_only set to zg, you must first locally save the configuration in the following path: 'openai/clip-vit-large-patch14'.cond_stage_model.transformerz#conditioner.embedders.0.transformerr.   cpur\   
embeddings"text_model.embeddings.position_ids)r   from_pretrained	Exception
ValueErrorr   r+   r   r   rd   rz   r   rg   re   r,   hasattrrs  position_idsr  load_state_dict)rj   rn  text_encoderconfig_namern   ctx
text_modelrz   text_model_dictremove_prefixesr[   prefix
param_nameparams                 r3   convert_ldm_clip_checkpointr  !  sm   5	#33KcF %<$=$= ;U&v.J U "

!"DO57\]O%F~~f%%<FOC$5$7 89 & 
   !0!6!6!8J'
JUS "9  
L11gj>S>S>`>`6a6a DdK""?3?  	/0@/A  Bi  j 	 Us   E EE
E.))positional_embeddingz/text_model.embeddings.position_embedding.weight)token_embedding.weightz,text_model.embeddings.token_embedding.weight)zln_final.weightz"text_model.final_layer_norm.weight)zln_final.biasz text_model.final_layer_norm.bias)text_projectionztext_projection.weight)	)z
resblocks.ztext_model.encoder.layers.)ln_1layer_norm1)ln_2layer_norm2)z.c_fc.z.fc1.)z.c_proj.z.fc2.)z.attnz
.self_attn)z	ln_final.z(transformer.text_model.final_layer_norm.)r  z8transformer.text_model.embeddings.token_embedding.weight)r  z;transformer.text_model.embeddings.position_embedding.weight|c                    [         R                  " SUS9n[        U5      n[        U R	                  5       5      n0 nU H-  nUR                  S5      (       d  M  X   XV[        S5      S  '   M/     UR                  R                  U5        U R                  5        VVs0 s H,  u  pxUR                  S5      (       d  M  U[        S5      S  U_M.     n	nn/ SQS/S	/S
/S/S/S.n
0 nU	R                  5        H  u  plUS [        S5       nUR                  U5      S   R                  S5      S   nUR                  U5      S   R                  U5      S   SS nX   n[        U5      n[        U5       H=  u  nnSR                  UUU/5      nUR                  S   U-  nUUU-  US-   U-   UU'   M?     M     UR                  R                  U5        UR                  R                  U S   U S   S.5        UR                   R                  U S   U S   S.5        ["        R$                  R'                  U S   5      UR(                  l        U$ s  snnf )Nrl  rm  rp  zcond_stage_model.transformer.zcond_stage_model.mapperzcond_stage_model.mapper.res)z
attn1.to_qz
attn1.to_kz
attn1.to_vzattn1.to_out.0r8   norm3zff.net.0.projzff.net.2)z
attn.c_qkvzattn.c_projr  r  zmlp.c_fcz
mlp.c_projzblocks.irW   r.   r   r(   zcond_stage_model.final_ln.biasz cond_stage_model.final_ln.weight)rB  r9  rT   rS   learnable_vector)r   ru  r&   rd   rz   r   rg   r   rz  re   r0   r   r/   rf   mapperrN  proj_outtorchnn	Parameteruncond_vectorr;  )rj   rn  rn   r   rz   r  r[   r   r!  keys_mapperMAPPINGmapped_weightsr\   r  suffixr  mapped_names
num_splitsr   mapped_namenew_namerf   s                         r3   #convert_paint_by_example_checkpointr  a  sp   --.M`pqF&v.E
!"DO>>899KU?O$C D FGH 
 
KK0
 $$&&DA<<12 	3#+
,
./2&   A()		$%!lG N!'')
&s:'6"2&,,S1"5yy $**6215a;}&
'5NA{xxf =>HKKNj0E',QY!a%5'IN8$ 6 * 
LL  0 
**?@ !CD	
 
NN""/ !23	
  %xx11*=O2PQELas   I=Ic                     [         R                  " U40 UDSU0D6n[	        5       (       a  [
        O[        nU" 5          U(       a  [        U5      O
[        U5      nS S S 5        [        U R                  5       5      n	/ n
US:X  aB  UR                  S:X  a2  X Vs/ s H  oR                  S5      (       d  M  UPM     sn-  n
U
S/-  n
0 nUS	-   U ;   a  [        XS	-      R                  S
   5      nOSnWR                  R                   R#                  S5      US'   U	 GH  nX;   a  M  U[%        U5      S  [&        ;   aN  UR)                  S	5      (       a  X   R*                  R-                  5       nOX   nX[&        U[%        U5      S     '   UR                  US-   5      (       d  M  U[%        US-   5      S  nUR)                  S5      (       ag  US [%        S5      *  n[.        R1                  S U5      nX   S U2S S 24   UUS-   '   X   XS-  2S S 24   UUS-   '   X   US-  S 2S S 24   UUS-   '   GM  UR)                  S5      (       aU  US [%        S5      *  n[.        R1                  S U5      nX   S U UUS-   '   X   XS-   UUS-   '   X   US-  S  UUS-   '   GM  [.        R1                  S U5      nX   UU'   GM     [	        5       (       a(  UR3                  5        H  u  nn[5        UUSUS9  M     U$ [7        US5      (       a$  [7        UR                   R8                  5      (       d  UR;                  SS 5        UR=                  U5        U$ ! [         a    [        SU SU S35      ef = f! , (       d  f       GN= fs  snf )Nrn  ro  zH, you must first locally save the configuration in the following path: 'z'.stabilityai/stable-diffusion-2   z/cond_stage_model.model.transformer.resblocks.23z&cond_stage_model.model.text_projectionr  r      ry  rt  ztransformer.z.in_proj_weightc                 Z    [         [        R                  " U R                  S5      5         $ Nr   	protectedreescapegroupms    r3   <lambda>.convert_open_clip_checkpoint.<locals>.<lambda>      	"))AGGTUJBW8Xr5   z.q_proj.weightr#   z.k_proj.weightz.v_proj.weightz.in_proj_biasc                 Z    [         [        R                  " U R                  S5      5         $ r  r  r  s    r3   r  r    r  r5   z.q_proj.biasz.k_proj.biasz.v_proj.biasc                 Z    [         [        R                  " U R                  S5      5         $ r  r  r  s    r3   r  r    r  r5   rq  rr  rs  )r   ru  rv  rw  r   r+   r   r   r   rd   rz   num_hidden_layersr   r   rf   r~  rs  
get_bufferrg   textenc_conversion_mapendswithT
contiguoustextenc_patternsubre   r,   rx  ry  r  rz  )rj   r|  r  has_projectionrn  config_kwargsrn   r}  r~  rz   keys_to_ignorer   r  r   r[   r\   new_keyr  r  s                      r3   convert_open_clip_checkpointr    s   
//p}p_op !8 9 9
{C	<J08P]^dPe
 
 
!"DN666;S;SWY;Ydndll;l.m1dnnCDDO!!Z/j*;!;<BB1EF<F<Q<Q<\<\<g<ghv<wO89 s6{}!77||-.."))446"JO23s6{}3EFG>>&>122#f~5689G 122!";S):%;$;<)--.XZab>HohwhXYk>Z*: :;>HogbcXcNcefNf>g*: :;>HogXYkm]^N^>_*: :;!!/22!"9S%9$9:)--.XZab<FOHW<U. 89<FOG`aVa<b. 89<FOGVWKM<Z. 89)--.XZab+5?(9 <   !0!6!6!8J'
JUS "9  
L11gj>S>S>`>`6a6a DdK""?3C  
+,<+=  >F  GR  FS  SU  V
 	

 
 os"   M M4N7NM14
Nc                 Z   U S   S   S   nUS   nUR                  S5      S   nUS:X  aL  UR                  R                  nUS:X  a"  [        5       n[        R
                  " S	US
9nXV4$ [        SU 35      eUS:X  a"  [        5       n[        R
                  " SUS
9nXV4$ [        SU 35      e)z
Returns the image processor and clip image encoder for the img2img unclip pipeline.

We currently know of two types of stable unclip models which separately use the clip and the openclip image
encoders.
r   r   embedder_configtargetr.   rW   ClipImageEmbedderzViT-L/14rl  rm  z<Unknown CLIP checkpoint name in stable diffusion checkpoint FrozenOpenCLIPImageEmbedderz%laion/CLIP-ViT-H-14-laion2B-s32B-b79KzAUnknown CLIP image embedder class in stable diffusion checkpoint )r0   r   r   r
   r   ru  r  )r   rn  image_embedder_configsd_clip_image_embedder_classclip_model_namefeature_extractorimage_encoders          r3   stable_unclip_image_encoderr    s     ,G4X>?PQ#8#B #?#E#Ec#J2#N #'::/66<<j( 2 49II/BRM  ++ &(detdu&vww	%)F	F.05EE3FV
 ++	 "OPlOmn
 	
r5   clip_stats_pathdevicec                    U S   S   S   nUS   nUR                  S5      S   nUS:X  a  UR                  nUR                  nUR                  R                  nUR                  R
                  n[        US9n[        XgS	9n	S
U;   aL  Uc  [        S5      e[        R                  " XS9u  pU
SSS24   n
USSS24   nU
US.nUR                  U5        X4$ [        SU 35      e)aQ  
Returns the noising components for the img2img and txt2img unclip pipelines.

Converts the stability noise augmentor into
1. a `StableUnCLIPImageNormalizer` for holding the CLIP stats
2. a `DDPMScheduler` for holding the noise schedule

If the noise augmentor config specifies a clip stats path, the `clip_stats_path` must be provided.
r   r   noise_aug_configr  r.   rW   CLIPEmbeddingNoiseAugmentation)embedding_dim)r   r   r  Nz6This stable unclip config requires a `clip_stats_path`map_location)meanstdzUnknown noise augmentor class: )r0   r   timestep_dimnoise_schedule_configr   r   r*   r   rw  r  loadrz  r  )r   r  r  r  noise_aug_classr  max_noise_levelr   image_normalizerimage_noising_scheduler	clip_meanclip_stdclip_stats_state_dicts                r3   &stable_unclip_image_noising_componentsr    s    'w/9:LM&x0O%++C04O::+22(55*@@JJ(>>LL6]S"/O"q 00& !YZZ"'**_"RI!$'*Ia(H "%!
 ,,-BC 44 "$COCT"UVVr5   c           	         [        XSS9nXHS'   UR                  S5        Ub  XhS'   Ub  XxS'   [        5       (       a  [        O[        n	U	" 5          [        S0 UD6n
S S S 5        SU ;   a  SnOSn[        U UUUSUS	9n[        5       (       a'  UR                  5        H  u  p[        W
US
US9  M     W
$ W
R                  U5        U
$ ! , (       d  f       Np= f)NT)r~   r   upcast_attentionr   r   r   r   F)r1   r	  r   r
  rq  rr   )
r   r  r   r+   r   r   r%  re   r,   rz  )rj   r   checkpoint_pathr~   r  r	  r   r   ctrlnet_configr}  r   r
  converted_ctrl_checkpointr  r  s                  r3   convert_controlnet_checkpointr  H  s     2/eijN)9%&}%(2G./&0C,- 7 9 9
{C	$6~6
 

 
*"&"' ; 7!   !:!@!@!BJ'
JUS "C
  	""#<=3 
s   C
Ccheckpoint_path_or_dictoriginal_config_fileprediction_type
model_typer	  scheduler_typenum_in_channelsr  from_safetensorsstable_unclipstable_unclip_priorr   adapterload_safety_checkersafety_checkerr  pipeline_classreturnc                 r   SSK JnJnJnJnJn Jn!Jn"Jn#J	n$J
n%Jn&Jn'  US:X  a  Sn[        U [        5      (       aL  U
(       a  SSKJn(  U(" U SS9n)ONU	c  [#        5       n	[$        R&                  " X	S	9n)O,[$        R&                  " X	S	9n)O[        U [(        5      (       a  U n)S
W);   a  U)S
   n*O[*        R-                  S5        Sn*SU);   a  U)S   n)SU);   a  M  Uc  Sn+Sn,Sn-UU!:H  n.Sn/Ub  SU;   a  US   nOSn/U+U);   a0  U)U+   R.                  S   S:X  a  Ub  SU;   a  US   nOSn/U*S:X  a  SnO/U,U);   a  Ub  SU;   a  US   nOSn/OU-U);   a  Ub  SU;   a  US   nOSn/U.(       a  Sn/U/b-  [1        [2        R4                  " U/[6        S9R8                  5      nOK[;        US5       n0U0R=                  5       nSSS5        O%[;        US5       n0U0R=                  5       nSSS5        [>        R@                  " U5      n1UcQ  SU1S    S!   ;   aE  U1S    S!   S   b9  U1S    S!   S   S"   RC                  S#5      S   n[*        R-                  S$U 35        O.Uc+  U1S    S!   S%   b  U1S    S!   S%   S!   S&   S':X  a  S(nOS)nUc  SnUc  US*;  a  U(       d  U OUnO
US(:X  a  U%OU#nUc  UUU$U"4;   a  S+nUc	  UU!:X  a  S,nOUc  S-nS.U1S    S!   ;   a  UU1S    S!   S.   S!   S/'   OS%U1S    S!   ;   a  UU1S    S!   S%   S!   S/'   S0U1S    S!   ;   a*  U1S    S!   S0   S1:X  a  Uc
  U*S2:X  a  S3OSnUc
  U*S2:X  a  S4OS5nO
Uc  S3nUc  S4nUc4  S6U1S    S!   ;   a(  [        U [        5      (       a  U OS7n2[E        U)U1U2X(U5      nS8U1S    S!   ;   a  U1S    S!   S8   n3OS9n3US*;   a'  S:S;S<S=U3S3S>S?SS@SASB.n4[F        RH                  " U45      n5SCnODSDU1S    S!   ;   a  U1S    S!   SD   n6OSEn6SFU1S    S!   ;   a  U1S    S!   SF   n7OSGn7[K        U7S:U6U3S@S?S?USH9n5U5RM                  S?SI9  USJ:X  a1  [)        U5RN                  5      n8SU8SK'   [P        RH                  " U85      n5OUSL:X  a!  [R        RH                  " U5RN                  5      n5OUSM:X  a!  [T        RH                  " U5RN                  5      n5OUSC:X  a!  [F        RH                  " U5RN                  5      n5OfUSN:X  a!  [V        RH                  " U5RN                  5      n5O?USO:X  a!  [X        RH                  " U5RN                  5      n5OUSP:X  a  U5n5O[[        SQU SR35      eUU!:X  a  U1S    S!   S.   S!   SS   n[]        U1UST9n9UU9SU'   [        U [        5      (       a  U OS7n2[_        U)U9U2USV9n:[a        5       (       a  [b        O[d        n;U;" 5          [g        S0 U9D6n<SSS5        [a        5       (       a-  US*;  a&  U:Ri                  5        H  u  n=n>[k        W<U=SU>SW9  M     OW<Rm                  U:5        Uc  Uc  [o        U1UST9n?[q        U)U?5      n@S U1;   a!  S!U1S    ;   a  SXU1S    S!   ;   a  U1S    S!   SX   nAOSYnAWAU?SZ'   [a        5       (       a  [b        O[d        n;U;" 5          [s        S0 U?D6nSSS5        [a        5       (       a'  W@Ri                  5        H  u  n=n>[k        UU=SU>SW9  M     O*URm                  W@5        OUc  [r        Rt                  " UUS[9nUS\:X  Ga  S]nBS^S_0nCUc  [w        U)WB4S`U0WCD6nDOUnD [x        Rt                  " S]SaUSb9nUc  U(       a'  U" UWDUW<U5UUUSe9nE[}        UESf5      (       a  S?WEl?        WE$ UU!:X  a9  [J        Rt                  " SgShSi9n5[        Rt                  " SgSjSi9nFU" UWDUW<U5UFUUSk9nEUE$ U" UWDUW<U5UUSl9nE[}        UESf5      (       a  S?WEl?        WE$ [        U1XSm9u  nGnHUSn:X  a  [        U15      u  nnIU&" UUIWGWHUWDW<U5USo9	nEUE$ USp:X  a  Ub  USq:X  az  SrnJ[        Rt                  " UJSsUSb9nK [x        Rt                  " StUS[9nL[        Rt                  " StUS[9nM[        Rt                  " WJSvUSb9nN[        RH                  " UNRN                  5      nNO[        SwU 35      eU'" WLWMWKWNWGWHUWDW<U5USx9nEUE$ [        SyU 35      eUSz:X  aD  [        U)5      nO [x        Rt                  " StUS[9n [        Rt                  " S{US[9nU" UWOW<U5SUS}9nEUE$ US~:X  a|  [        U)UUS9nD Uc  [x        Rt                  " StUS[9OUnU(       a*  [        Rt                  " S{US[9n[        Rt                  " S{US[9nU(       a  U" UWDUW<UU5UUS9nEUE$ U" UWDUW<U5UUSl9nE WE$ US*;   Ga%  US):H  nPUPS?L a  Uc   [x        Rt                  " StUS[9nWPS?L a  Uc
  [        U)US[9nUc   [x        Rt                  " SSUS9nUc"  SnBSS0nCWP(       a  SOSnQ[w        U)WB4UQSUS.WCD6n[a        5       (       a&  U:Ri                  5        H  u  n=n>[k        W<U=SU>SW9  M     U(       a  U" UUUUUW<UU5SS9	nEUE$ U(       a  U" UUUUUW<UU5SS9	nEUE$ UUUUUW<U5S.nRUU#:X  d  UU$:X  a  WRR                  SWP05        WP(       a  WRR                  SS?05        U" S0 WRD6nE WE$ [        U15      nS[        U)US5      nD[        Rt                  " SUS[9nU" UUDUW<U5S9nEWE$ ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNG= f! , (       d  f       GN= f! [z         a    [[        ScU Sd35      ef = f! [z         a    [[        ScU Su35      ef = f! [z         a    [[        ScU Su35      ef = f! [z         a    [[        ScU S|35      ef = f! [z         a    [[        ScU Su35      ef = f! [z         a    [[        ScU Su35      ef = f! [z         a    [[        ScU S35      ef = f)a  
Load a Stable Diffusion pipeline object from a CompVis-style `.ckpt`/`.safetensors` file and (ideally) a `.yaml`
config file.

Although many of the arguments can be automatically inferred, some of these rely on brittle checks against the
global step count, which will likely fail for models that have undergone further fine-tuning. Therefore, it is
recommended that you override the default values and/or supply an `original_config_file` wherever possible.

Args:
    checkpoint_path_or_dict (`str` or `dict`): Path to `.ckpt` file, or the state dict.
    original_config_file (`str`):
        Path to `.yaml` config file corresponding to the original architecture. If `None`, will be automatically
        inferred by looking for a key that only exists in SD2.0 models.
    image_size (`int`, *optional*, defaults to 512):
        The image size that the model was trained on. Use 512 for Stable Diffusion v1.X and Stable Diffusion v2
        Base. Use 768 for Stable Diffusion v2.
    prediction_type (`str`, *optional*):
        The prediction type that the model was trained on. Use `'epsilon'` for Stable Diffusion v1.X and Stable
        Diffusion v2 Base. Use `'v_prediction'` for Stable Diffusion v2.
    num_in_channels (`int`, *optional*, defaults to None):
        The number of input channels. If `None`, it will be automatically inferred.
    scheduler_type (`str`, *optional*, defaults to 'pndm'):
        Type of scheduler to use. Should be one of `["pndm", "lms", "heun", "euler", "euler-ancestral", "dpm",
        "ddim"]`.
    model_type (`str`, *optional*, defaults to `None`):
        The pipeline type. `None` to automatically infer, or one of `["FrozenOpenCLIPEmbedder",
        "FrozenCLIPEmbedder", "PaintByExample"]`.
    is_img2img (`bool`, *optional*, defaults to `False`):
        Whether the model should be loaded as an img2img pipeline.
    extract_ema (`bool`, *optional*, defaults to `False`): Only relevant for
        checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights or not. Defaults to
        `False`. Pass `True` to extract the EMA weights. EMA weights usually yield higher quality images for
        inference. Non-EMA weights are usually better to continue fine-tuning.
    upcast_attention (`bool`, *optional*, defaults to `None`):
        Whether the attention computation should always be upcasted. This is necessary when running stable
        diffusion 2.1.
    device (`str`, *optional*, defaults to `None`):
        The device to use. Pass `None` to determine automatically.
    from_safetensors (`str`, *optional*, defaults to `False`):
        If `checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.
    load_safety_checker (`bool`, *optional*, defaults to `True`):
        Whether to load the safety checker or not. Defaults to `True`.
    safety_checker (`StableDiffusionSafetyChecker`, *optional*, defaults to `None`):
        Safety checker to use. If this parameter is `None`, the function will load a new instance of
        [StableDiffusionSafetyChecker] by itself, if needed.
    feature_extractor (`AutoFeatureExtractor`, *optional*, defaults to `None`):
        Feature extractor to use. If this parameter is `None`, the function will load a new instance of
        [AutoFeatureExtractor] by itself, if needed.
    pipeline_class (`str`, *optional*, defaults to `None`):
        The pipeline class to use. Pass `None` to determine automatically.
    local_files_only (`bool`, *optional*, defaults to `False`):
        Whether or not to only look at local files (i.e., do not try to download the model).
    vae (`AutoencoderKL`, *optional*, defaults to `None`):
        Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. If
        this parameter is `None`, the function will load a new instance of [CLIP] by itself, if needed.
    text_encoder (`CLIPTextModel`, *optional*, defaults to `None`):
        An instance of [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel)
        to use, specifically the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)
        variant. If this parameter is `None`, the function will load a new instance of [CLIP] by itself, if needed.
    tokenizer (`CLIPTokenizer`, *optional*, defaults to `None`):
        An instance of
        [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer)
        to use. If this parameter is `None`, the function will load a new instance of [CLIPTokenizer] by itself, if
        needed.
    config_files (`Dict[str, str]`, *optional*, defaults to `None`):
        A dictionary mapping from config file names to their contents. If this parameter is `None`, the function
        will load the config files by itself, if needed. Valid keys are:
            - `v1`: Config file for Stable Diffusion v1
            - `v2`: Config file for Stable Diffusion v2
            - `xl`: Config file for Stable Diffusion XL
            - `xl_refiner`: Config file for Stable Diffusion XL Refiner
    return: A StableDiffusionPipeline object representing the passed-in `.ckpt`/`.safetensors` file.
r   )LDMTextToImagePipelinePaintByExamplePipeline!StableDiffusionControlNetPipelineStableDiffusionInpaintPipelineStableDiffusionPipelineStableDiffusionUpscalePipeline*StableDiffusionXLControlNetInpaintPipeline StableDiffusionXLImg2ImgPipeline StableDiffusionXLInpaintPipelineStableDiffusionXLPipelineStableUnCLIPImg2ImgPipelineStableUnCLIPPipelinezv-predictionv_prediction)	load_filerq  )r  Nr  global_stepz"global_step key not found in model
state_dictzMmodel.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_k.weightzEconditioner.embedders.1.model.transformer.resblocks.9.mlp.c_proj.biaszEconditioner.embedders.0.model.transformer.resblocks.9.mlp.c_proj.biasv1zjhttps://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yamlrW   r  v2zphttps://raw.githubusercontent.com/Stability-AI/stablediffusion/main/configs/stable-diffusion/v2-inference-v.yamli Txlzghttps://raw.githubusercontent.com/Stability-AI/generative-models/main/configs/inference/sd_xl_base.yaml
xl_refinerzjhttps://raw.githubusercontent.com/Stability-AI/generative-models/main/configs/inference/sd_xl_refiner.yamlznhttps://raw.githubusercontent.com/Stability-AI/stablediffusion/main/configs/stable-diffusion/x4-upscaling.yaml)timeoutrr   r   r   r  r.   z1no `model_type` given, `model_type` inferred as: r   r   r   SDXLSDXL-Refiner)r
  r  	      rb   r   r   parameterizationr!  iY epsilon   i   r   r   r   i  r   g_QK?g~jt?linearg      ?Fr(   leading)r   r   r   interpolation_typer   r  sample_max_valueset_alpha_to_oneskip_prk_stepssteps_offsettimestep_spacingeulerr   g{Gz?r   g(\µ?)r   r   r   r   r  clip_sampler  r  )r  pndmr  lmsheunzeuler-ancestraldpmddimzScheduler of type z doesn't exist!r~   )r~   r  )r1   r	  rr  scale_factorg{P?scaling_factorrm  FrozenOpenCLIPEmbedderr  	subfolderr{  rn  	tokenizer)r#  rn  ro  zd, you must first locally save the tokenizer in the following path: 'stabilityai/stable-diffusion-2'.)vaer{  r$  unet	schedulerr   r  r  requires_safety_checkerz(stabilityai/stable-diffusion-x4-upscalerr'  )r#  low_res_scheduler)r%  r{  r$  r&  r'  r)  r  r  )r%  r{  r$  r&  r'  r  r  )r  r  img2img)	r  r  r  r  r$  r{  r&  r'  r%  txt2imgkarlozkakaobrain/karlo-v1-alphapriorrl  zc, you must first locally save the tokenizer in the following path: 'openai/clip-vit-large-patch14'.prior_schedulerz'unknown prior for stable unclip model: )prior_tokenizerprior_text_encoderr-  r.  r  r  r$  r{  r&  r'  r%  zunknown `stable_unclip` type: PaintByExamplez'CompVis/stable-diffusion-safety-checkerzu, you must first locally save the feature_extractor in the following path: 'CompVis/stable-diffusion-safety-checker'.)r%  r  r&  r'  r  r  FrozenCLIPEmbedder)rn  r{  )r%  r{  r$  r&  r   r'  r  r  z(laion/CLIP-ViT-bigG-14-laion2B-39B-b160k!)	pad_tokenrn  z, you must first locally save the tokenizer in the following path: 'laion/CLIP-ViT-bigG-14-laion2B-39B-b160k' with `pad_token` set to '!'.projection_dimr   zconditioner.embedders.0.model.zconditioner.embedders.1.model.)r  r  rn  )	r%  r{  r$  text_encoder_2tokenizer_2r&  r   r'  force_zeros_for_empty_prompt)	r%  r{  r$  r6  r7  r&  r  r'  r8  )r%  r{  r$  r6  r7  r&  r'  requires_aesthetics_scorer8  zbert-base-uncased)vqvaebertr$  r&  r'  r  )O	diffusersr  r  r  r  r  r  r  r  r  r  r  r  rc   r  safetensors.torchr  r"   r  r  dictr  debugrf   r   requestsr+  r!   contentopenreadyaml	safe_loadr0   r  r   from_configr   register_to_configrn   r   r   r   r   r   rw  r   r%  r   r+   r   r   re   r,   rz  r   r4  r   ru  r  r   rv  rx  r(  r   r  r  r   r   r   r  r  r   r  r)   updater   rj  r	   )Tr  r  r~   r  r  r	  r  r  r  r  r  r  r  r  r   r  r  r  r  r  rn  vae_pathr%  r{  r6  r$  r7  config_filesr  r  r  r  r  r  r  r  r  r  r  r  rE  rj   r  key_name_v2_1key_name_sd_xl_basekey_name_sd_xl_refiner
is_upscale
config_urlfr   r1   r   scheduler_dictr'  r   r   rn   r   converted_unet_checkpointr}  r&  r  r  
vae_configconverted_vae_checkpointvae_scaling_factorr|  r  r~  piper)  r  r  r  karlo_modelr-  r/  prior_text_modelr.  vision_model
is_refinerr  pipeline_kwargstext_configsT                                                                                       r3   ,download_from_original_stable_diffusion_ckptr]  z  s   R    .(()3//@"#:5IJ~#"ZZ(?U
"ZZ(?U
	+T	2	2,
 
" /9: *
$-
 *
$ #ge!h#'EE

 #(<#/#5  FJJ&:m+D+J+J2+NRV+V'DL,@'3D'9$ P
f$#'  J.'DL,@'3D'9$ G
#z1'LL,H'3L'A$ J
 JJ!#*8<<
Le+f+n+n#o *C0A'(vvx$ 10 &,#$668  - nn%9:O 	?7#;H#EEG$X./BCO$W-h78KLXV\\]`abde
HUV		 8 BCS T `7#H-.>?I-X\``J'JJ55<F4LmN:D:N6TtN>&(26 $
 >5S#S		 0::Ud *=9(CMR	_W5h?	?Xg *+;<XF}U 	og6x@@G$X./ABcI" ,7&+@inO !,v 53J"'OJ48PQY8ZZ*45Lc*R*R&XZ2z[

 og.x88-g6x@M"--,!"*#6( # %" )
 +66~F	 _W5h??(1(;NKJJ?73H==&w/9,GHH!)! 3"+	
	   U 3i&&'#' !--f5		5	 (44Y5E5EF		6	!)55i6F6FG		7	"*66y7G7GH		,	,3??	@P@PQ		5	 /;;I<L<LM		6	!	-n-=_MNN77$W-h7FxPQ]^
 /:VK&6K"#&01H#&N&N"TVD ;Kd! !8 9 9
{C	#2k2 
   55%>%D%D%F!
E+D*e5Q &G 	67 CK0ZX
#=j*#U  &OG44/'":8"DD!0!9(!CN!S!('9
#$$;$=$= ;U-*-C  #$$%=%C%C%E!
E+CU%P &F  89	++HGWX--6$n55K:JN[J &J	%550KZjI  %!+'')#1&7	 4!:;;38D0V KS  #AA)99>+	 %2$A$A>J]%! &!+''&7#1&7	B Kk &!+''#1&7 4!:;;38D0V KQ 9_955 	)3N3_0!=2&7"/%5,C'!+'B Ke )+&.2E2P"=K,<<#wIYE*7*G*G;N^+ (C'R'R7JZ($ '6&E&E#/@Sc'O '4&?&?@V@V&WO-0WXkWl.mnn+$3'7$3%5,C'!+'p KM *,J=/*Z[[	'	'::F	%55/BRI	 4 D D9L\! &&/
f KW 
+	+0)9

		 $ --.M`pq  9II9L\N !5 D D9L\! !'#%#-"3	Df KQ "'##-"3DP K 
/	/>1
%i&7)993FV	 %l&:6zTdeL+;;>#`p !DK-t4M9C5IiF9 #!1  N #$$%>%D%D%F!
E+D*e5Q &G !)#-'%#-1
Dd KO !)#-'#-1
DL K1  ,&"0*&O "BB"BB&&(CZ'PQ&&(F'NO!4O4D K -_=0[I
%556I\lm	%CjI\`luvKK 10 -,` 
6 U4  	/0@/A  Bf  g 	l % (;<L;M  Nq  r P  	/0@/A  Be  f 	  	/0@/A  Bw  x 	,  	/0@/A  Be  f 	T   34D3E  Fi  j    34D3E  FP  Q s   j&j8>k
k"k. 'l l( m m" m? n &
j58
k

k
k+.ll%(mm"m<?nn6r  r   r   c
                 \   U(       aD  SSK Jn
  0 nU
" U SSS9 nUR                  5        H  nUR                  U5      X'   M     S S S 5        O6Uc  [	        5       n[
        R                  " XS9nO[
        R                  " XS9nSU;   a  US   nSU;   a  M  [        US5       nUR                  5       nS S S 5        [        R                  " U5      nUb  XNS	   S
   S   S
   S'   SUS	   S
   ;  a  [        S5      e[        UUU UUUUU	S9nU$ ! , (       d  f       N= f! , (       d  f       Nq= f)Nr   )	safe_openptrq  )	frameworkr  r  r  r	  r   r   r   r   r   z5`control_stage_config` not present in original config)r   r   )safetensorsr_  rz   
get_tensorr"   r  r  rB  rC  rD  rE  rw  r  )r  r  r~   r	  r  r  r  r  r   r   r_  rj   rP  r[   r   r   s                   r3   &download_controlnet_from_original_ckptrd  !  s;    )
$uEvvx"#,,s"3
   FE >\FOIJOIJ *
$-
 *
$ 
"C	(A vvx 
)nn%9:O"Ud *=9(CMR_W%=h%GGPQQ.3/	J I FE 
)	(s   +D!D
D
D+)r(   )r   )NNN)F)NFFF)FN)zcond_stage_model.model.FF)NN)NNNNFr  NNNFNNNNNTNNNFNNNNNNN)r  FNNNFNN)b__doc__r  
contextlibr   ior   typingr   r   r   r@  r  rD  transformersr   r	   r
   r   r   r   r   r   r   modelsr   r   r   r   
schedulersr   r   r   r   r   r   r   r   r   utilsr   r    utils.constantsr!   utils.torch_utilsr"   *latent_diffusion.pipeline_latent_diffusionr$   r%   paint_by_exampler&   pipeline_utilsr'   r  r)   stable_unclip_image_normalizerr*   
accelerater+   accelerate.utilsr,   
get_logger__name__r  r4   rK   rN   rQ   rU   rw   r}   r   r   r   r   r   r%  r4  rj  r  textenc_conversion_lstr  "textenc_transformer_conversion_lstr  r  compiler/   rz   r  r  r  r  r  r  r  Tensorboolr]  rd  )xs   0r3   <module>r}     s   > 	 "  ( (   
 
 
 
 
 
 6 8 + T 9 . 8 G -<			H	%C, *> lp4?n	;jc jZS < afuphV/d$N  /EE.DA$!*.DE & " -OO,NqRYYqt_ad",NO	**SXXinn&678@L %Od",L UY*5&.sm*5DLSM*5h /h !% $ %)'+"#')-%)!%" $=A8<(,9d
"3S%,,->(?#?@d
d
 d
 	d

 d
 d
 d
 c]d
 tnd
 d
 d
 C=d
 "#d
 c]d
 d
  d^!d
" #d
$ 9:%d
&   45'd
( &)d
: ;d
T %)'+",0*.444 4 	4
 c]4 tn4 4 4 $D>4 "$4 4e F Ps   I-"I!