
    +hf                        S SK r S SKJr  S SKrSSKJrJrJr  \R                  " \	5      r
S rSS jrSS jrS rS	 rS
 rS rS rS\S\\   S\4S jrS rS rS rS rS rS rSS jrSS jrS rg)    N)List   )is_peft_versionloggingstate_dict_all_zeroc                 V    U R                  SSS9u  p[        R                  " X!/SS9nU$ )Nr   r   dim)chunktorchcat)weightshiftscale
new_weights       a/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/loaders/lora_conversion_utils.pyswap_scale_shiftr      s.    <<q<)LEE>q1J    c                 	  ^ [        U R                  5       5      n/ SQn/ SQnSnSnU H?  m[        U4S jU 5       5      (       a  SnM!  [        U4S jU 5       5      (       d  M=  SnMA     U(       a?  U(       a8  U H0  m[        U4S jU 5       5      (       d  M  U R                  T5        M2     U $ Sn	U H!  m[        U4S jU 5       5      (       d  M  Sn	  O   U	(       d  U $ 0 n
/ S	Qn[	        5       [	        5       [	        5       pnU H  nS
U;   a  U R                  U5      X'   M  [        UR                  U5      S U S   5      nUS   U;   a  UR                  U5        MZ  US   U;   a  UR                  U5        Mv  US   U;   a  UR                  U5        M  [        SU S35      e   U VVs0 s H$  nUU  Vs/ s H  nSU U 3U;   d  M  UPM     sn_M&     nnnU VVs0 s H$  nUU  Vs/ s H  nSU U 3U;   d  M  UPM     sn_M&     nnnU VVs0 s H$  nUU  Vs/ s H  nSU U 3U;   d  M  UPM     sn_M&     nnnU H  nUS-
  UR                  S-   -  nUS-
  UR                  S-   -  nUU    H  m[        TR                  U5      U   5      nST;  a  UU   OSnST;  a  [        U5      OSnUR                  TR                  U5      S US-
   [        U5      UU/-   TR                  U5      US-   S  -   5      nU R                  T5      U
U'   M     M     U H  nS nUS:X  a  US   S/nO+US:X  a  US   S/nOUS:X  a  US   S/nO[        SU S35      eUU    HU  mUR                  TR                  U5      S US-
   U-   TR                  U5      US  -   5      nU R                  T5      U
U'   MW     M     U H  nUUR                  S-   -  nUUR                  S-   -  nUU    H  m[        TR                  U5      U   5      nUU   nUS:  a  [        U5      OSnUR                  TR                  U5      S US-
   [        U5      UU/-   TR                  U5      US-   S  -   5      nU R                  T5      U
U'   M     M     U (       a  [        S5      eU
$ s  snf s  snnf s  snf s  snnf s  snf s  snnf )N)input_blocksmiddle_blockoutput_blocks)down_blocks	mid_block	up_blocksFc              3   ,   >#    U  H	  oT;   v   M     g 7fN .0pkeys     r   	<genexpr>5_maybe_map_sgm_blocks_to_diffusers.<locals>.<genexpr>*        .ACx   Tc              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   r   s     r   r#   r$   ,   s     4#3ac#3r&   c              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   r   s     r   r#   r$   3   s     2\8\r&   c              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   r   s     r   r#   r$   :   r%   r&   )resnets
attentions
upsamplerstextr      r   z'Checkpoint not supported because layer z not supported.r   r   r   opdownsamplers01zInvalid middle block id .z:At this point all state dict entries have to be converted.)listkeysanypopsetintsplitadd
ValueErrorlayers_per_blockstrjoin)
state_dictunet_config	delimiterblock_slice_posall_keyssgm_patternsnot_sgm_patternscontains_sgm_patternscontains_not_sgm_patternsis_in_sgm_formatnew_state_dictinner_block_mapinput_block_idsmiddle_block_idsoutput_block_idslayerlayer_idr"   r   middle_blocksr   iblock_idlayer_in_block_idinner_block_idinner_block_keyinner_layers_in_blocknew_keykey_parts                    `           r   "_maybe_map_sgm_blocks_to_diffusersr[       sg   JOO%&HDL@ " %....$(!4#3444(,%	  !:C2\222s#   ....# 
  N=O ;>%'7OU?$.NN5$9N!5;;y12B?CBGHHA%'##H-aE) $$X.aE) $$X. #J5'Q`!abb   ('H 	*\*3,yk(0TX[0[3*\\'   )(H 	*\*3,yk(0TX[0[3*\\(   )(H 	*]*3-	{8*0UY\0\3*]](   E{;;a?@U{'C'Ca'GH?C 9!5o!FGNAESon=VdO>B#oC(9$:SV!nn		)$%:':;x=/3HIJ))I&':'<=>G
 '1nnS&9N7# #	  6'*C0H!V'*C0H!V'*C0H7s!<== #Cnn		)$%:':;hFS\I]^m^nIooG '1nnS&9N7#	 $ " 559:!=!=!AB #C 9!5o!FGN-n=O>Lq>PC(9$:VY!nn		)$%:':;x=/3HIJ))I&':'<=>G
 '1nnS&9N7# $	  UVV ]
 ]
 ^sZ   1R,<R'R'R,$R7/R2R2R7S"R=5R=;S'R,2R7=Sc                    0 n0 n0 n0 n[        S U  5       5      n[        S U  5       5      n[        S U  5       5      n	U(       d  U(       d  U	(       a  [        SS5      (       a  [        S5      e[        U R	                  5       5      n
U
 GHj  nUR                  S5      (       d  M  UR                  S5      S	   nUS
-   nUS-   nUR                  S5      (       a  [        U5      nU R                  U5      X?'   U R                  U5      X?R                  SS5      '   U(       a=  SU;   a  SOSnU R                  UR                  SS5      5      X?R                  US5      '   GOPUR                  S5      (       Ga9  [        X5      nUR                  S5      (       a7  U R                  U5      XO'   U R                  U5      XOR                  SS5      '   O6U R                  U5      X_'   U R                  U5      X_R                  SS5      '   U(       d  U	(       a  SU;   a  SOSnUR                  S5      (       a4  U R                  UR                  SS5      5      XOR                  US5      '   OIUR                  S5      (       a3  U R                  UR                  SS5      5      X_R                  US5      '   X;   d  GM/  U R                  U5      R                  5       nUR                  [        UWU5      5        GMm     [        U 5      S	:  a+  [        SSR!                  U R	                  5       5       35      e["        R%                  S5        UR'                  5        VVs0 s H  u  nnU SU 3U_M     nnnUR'                  5        VVs0 s H  u  nnU SU 3U_M     nnn[        U5      S	:  a*  UR'                  5        VVs0 s H  u  nnSU 3U_M     snnOSnUb  UR                  U5        0 UEUEnUU4$ s  snnf s  snnf s  snnf )a  
Converts a non-Diffusers LoRA state dict to a Diffusers compatible state dict.

Args:
    state_dict (`dict`): The state dict to convert.
    unet_name (`str`, optional): The name of the U-Net module in the Diffusers model. Defaults to "unet".
    text_encoder_name (`str`, optional): The name of the text encoder module in the Diffusers model. Defaults to
        "text_encoder".

Returns:
    `tuple`: A tuple containing the converted state dict and a dictionary of alphas.
c              3   D   #    U  H  nS U;   =(       a    SU;   v   M     g7f)
dora_scale
lora_unet_Nr   r    ks     r   r#   ;_convert_non_diffusers_lora_to_diffusers.<locals>.<genexpr>   s$     [PZ1|q0F\Q5FFPZ    c              3   \   #    U  H"  nS U;   =(       a    SU;   =(       d    SU;   v   M$     g7f)r^   lora_te_	lora_te1_Nr   r`   s     r   r#   rb      s/     mbl]^\Q.XJ!O4W{VWGWXbl   *,c              3   D   #    U  H  nS U;   =(       a    SU;   v   M     g7f)r^   	lora_te2_Nr   r`   s     r   r#   rb      s"     Yjla/DK14DDjrc   <z0.9.0zeYou need `peft` 0.9.0 at least to use DoRA-enabled LoRAs. Please upgrade your installation of `peft`.lora_down.weightr4   r   .lora_up.weight.alphar_   .down..up.z_lora.down.z.lora.down.r^   z.lora_magnitude_vector.)re   rf   ri   re   rf   .lora_linear_layer.ri   z7The following keys have not been correctly renamed: 

 , z"Non-diffusers checkpoint detected.text_encoder_2.N)r7   r   r=   r5   r6   endswithr;   
startswith_convert_unet_lora_keyr8   replace_convert_text_encoder_lora_keyitemupdate_get_alpha_namelenr@   loggerinfoitems)rA   	unet_nametext_encoder_nameunet_state_dictte_state_dictte2_state_dictnetwork_alphasdora_present_in_unetdora_present_in_tedora_present_in_te2all_lora_keysr"   	lora_namelora_name_uplora_name_alphadiffusers_namedora_scale_key_to_replacedora_scale_key_to_replace_tealphamodule_nameparamsrK   s                         r   (_convert_non_diffusers_lora_to_diffusersr      s    OMNN [PZ[[mblmmYjYY15H3((w 
 *+M||.// IIcN1%	 !#44#h. --3C8N /9nnS.AO+HRWcHdO228VDE $=Jn=\Mbo)NN3;;/A<#PQ   6 67PRk lm
 !!"HII;CKN ##$=>>0:s0C-JT..YeJf44XvFG1;1D.KU>>ZfKg55hGH "%8%2n%DMJ_ - ''(ABB"s{{3E|'TU ""8"89UWp"qr ))+66"s{{3E|'TU ##9#9:VXq#rs
 (NN?388:E!!//>SX"YZo t :TUYU^U^_i_n_n_pUqTrstt
KK45 XgWlWlWnoWn@SV)Ak]3V;WnOo]j]p]p]rs]rFYkSY)*!K=96A]rMs ~" M[L`L`LbcLb5H[&?;-	(&	0Lbc 
 !^,99=9N>)) pscs   O8O>Pc                 x  ^ U R                  SS5      R                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS	5      mTR                  S
S	5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mTR                  SS5      mST;   a  ST;  a  S n[        R                  " USTS!S"9mS#T;   a  TR                  S$S%5      mS&T;   a  TR                  S'S(5      mS)T;   d  S*T;   a  TR                  S+S,5      mS-T;   a  TR                  S.S/5      mST;   a  TR                  SS5      mS0T;   a  TR                  S0S/5      mST;   a;  S1T;   d  S2T;   a&  TR                  S1S35      mTR                  S2S45      mT$ S5T;   a   T$ [        U4S6 jS7 5       5      (       a   T$  T$ )8z:
Converts a U-Net LoRA key to a Diffusers compatible key.
r_    _r4   zinput.blocksr   zdown.blockszmiddle.blockr   z	mid.blockzoutput.blocksr   z	up.blocksztransformer.blockstransformer_blocksz	to.q.lora	to_q_loraz	to.k.lora	to_k_loraz	to.v.lora	to_v_lorazto.out.0.lorato_out_lorazproj.inproj_inzproj.outproj_outz
emb.layerstime_emb_projembztime.emb.projz\.\d+(?=\D*$)r/   )countz.in.zin.layers.2conv1z.out.zout.layers.3conv2r1   r,   r0   convskipzskip.connectionconv_shortcutzconv.shortcutattn1attn2zattn1.processorzattn2.processorffc              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   )r    r"   r   s     r   r#   )_convert_unet_lora_key.<locals>.<genexpr>5  s     F.EsN".Er&   )r   r   )rw   resubr7   )r"   patternr   s     @r   rv   rv     sx    [[r2::3DN $++NMJN#++M=IN#++NKHN#++KEN#++O[IN#++KEN#++,@BVWN#++KEN#++KEN#++KEN#++O]KN#++IyAN#++J
CN#++L/JN ?.#H"^1E'//wG. '//H'<>+I'//f='//0A?S .('//Q.('//Q ~-n$>(A+33G=NON+33G=NON  ^#  
F.EF	F	F  	r   c                    UR                  S5      (       a  UR                  S5      (       a  SOSnOSnU R                  US5      R                  SS5      nUR                  SS	5      nUR                  S
S5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nSU;   d  SU;   a   U$ SU;   a  UR                  SS5      nU$ )zA
Converts a text encoder LoRA key to a Diffusers compatible key.
rp   re   rf   ri   r   r   r4   z
text.model
text_modelz	self.attn	self_attnzq.proj.lorar   zk.proj.lorar   zv.proj.lorar   zout.proj.lorar   ztext.projectiontext_projectionmlp.lora.rq   )ru   rw   )r"   r   key_to_replacer   s       r   rx   rx   =  s    566'0';';J'G'G[$[[4<<S#FN#++L,GN#++KEN#++M;GN#++M;GN#++M;GN#++O]KN#++,=?PQNn$(9^(K  
.	  (//:OPr   c                     U R                  S5      (       a  SnOU R                  S5      (       a  SnOSnX1R                  S5      S   -   S-   nXB0$ )	z6
Gets the correct alpha name for the Diffusers model.
r_   zunet.rp   text_encoder.rs   r   r   rm   )ru   r;   )r   r   r   prefixnew_names        r   r{   r{   Y  s\     !!,//		#	#$=	>	> ",,X6q99HDHr   c                   ^^ S mS*S jmUU4S jnS n[        S U  5       5      nU(       a_  U R                  5        VVs0 s H@  u  pEUR                  S5      (       d  M  UR                  SS5      R                  S	S
5      U_MB     n nnU $ [        S U  5       5      nU(       GdX  U R                  5        VVs0 s H  u  pEUR                  SS5      U_M     n nnU R                  5        VVs0 s H  u  pEUR                  SS5      U_M     n nn[        S U  5       5      nU(       aj  [	        U S5      nU(       a  [
        R                  S5        O[
        R                  S5        U R                  5        VVs0 s H  u  pESU;  d  M  XE_M     n nn[        S U  5       5      n	U	(       az  [	        U S5      n
U
(       a  [
        R                  S5        O[
        R                  S5        U R                  5        VVs0 s H  u  pEUR                  S5      (       a  M  XE_M!     n nn[        S U  5       5      nU(       aj  [	        U S5      nU(       a  [
        R                  S5        O[
        R                  S5        U R                  5        VVs0 s H  u  pESU;  d  M  XE_M     n nn[        S U  5       5      nU(       ar  [	        U S5      nU(       a  [
        R                  S5        O[
        R                  S 5        U R                  5        VVs0 s H  u  pES!U;  d  M  SU;  d  M  XE_M     n nnS"S#/n[        S$ U  5       5      (       a  UR                  S%5        U R                  5        VVs0 s H)  u  pEUR                  S&5      (       d  M  [        XO5      U_M+     n nn[        S' U  5       5      (       aA  [
        R                  S(5        U R                  5        VVs0 s H  u  pES)U;  d  M  XE_M     n nnU(       a  U" U 5      $ U" U 5      $ s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf s  snnf )+Nc                    US-   U ;  a  g U R                  US-   5      nUR                  S   n[        R                  " XTR                  UR
                  SS9nU R                  US-   U5      R                  5       nXu-  nUn	Sn
U	S-  U
:  a  U	S-  n	U
S-  n
U	S-  U
:  a  M  XI-  XS-   '   U R                  US	-   5      U
-  XS
-   '   g )N.lora_down.weightr   Fdtypedevicerequires_gradrm         ?r   .lora_A.weightrl   .lora_B.weight)r8   shaper   tensorr   r   ry   )sds_sdait_sdsds_keyait_keydown_weightrankdefault_alphar   r   
scale_downscale_ups              r   _convert_to_ai_toolkitE_convert_kohya_flux_lora_to_diffusers.<locals>._convert_to_ai_toolkitj  s    ((6jj+>!>?   #T1B1B;K]K]mrs

7X-}=BBD 
1nx'!OJMH 1nx' .9-E))*-3ZZBS8S-TW_-_))*r   c                    US-   U ;  a  g U R                  US-   5      nU R                  US-   5      nUR                  S   n[        R                  " XuR                  UR
                  SS9nU R                  US-   U5      n	X-  n
U
nSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  X[-  nXl-  n[        U5      nUc  UR                  S   U-  /U-  nO[        U5      UR                  S   :X  d   eSnX}-  S:X  a  X}-  nS	nSn[        [        U5      5       Hh  n[        [        U5      5       HE  nUU:X  a  M  U=(       a1    [        R                  " UUUUU   -   2UU-  US
-   U-  24   S:H  5      nMG     UUU   -  nMj     U(       a  [        R                  SU 35        U Vs/ s H  nUS-   PM
     nnU Vs/ s H  nUS-   PM
     nnU(       dk  UR                  [        R                  UU5      5        UR                  [        U[        R                   " XdSS95       VVs0 s H	  u  nnUU_M     snn5        g UR                  [        U[        R"                  " X]SS95       VVs0 s H	  u  nnUU_M     snn5        Sn[        [        U5      5       H9  nUUUUU   -   2UW-  US
-   U-  24   R%                  5       UUU   '   UUU   -  nM;     g s  snf s  snf s  snnf s  snnf )Nr   rl   r   Fr   rm   r   r   Tr/   zweight is sparse: r   r   r	   )r8   r   r   r   r   r   r|   sumrangeallr}   r~   rz   dictfromkeyszipr;   r   
contiguous)r   r   r   ait_keysdimsr   	up_weightsd_lora_rankr   r   r   r   r   
num_splits	is_sparseait_rankrS   jra   ait_down_keysait_up_keysvs                         r   _convert_to_ai_toolkit_catI_convert_kohya_flux_lora_to_diffusers.<locals>._convert_to_ai_toolkit_cat  s$   ((6jj+>!>?JJw)::;	"((+  1 1+:L:L\a
 

7X-}=$ 
1nx'!OJMH 1nx' ".(	 ]
<OOA&*45
BDt9	 2222 	$)#1HIA3t9%s4y)AAv  ) !eii!!a$q'k/1x<1q5HBT3T"TUYZZ/I * T!W & 0	:; 8@@x!--x@5=>Xq++X>MM$--{CD MMCU[[^_=`,ab,aDAq1a4,abc MMCu{{;hi?j,kl,kDAq1a4,klm A3t9%)21q47{?ALTUXYTY]eSeDe3e)f)q)q)s{1~&T!W & A> c ms   'K<K&K"
,K(
c                   >^ 0 m[        S5       H  nT" U TSU S3SU S35        T" U TSU S3SU S3SU S3SU S	3/5        T" U TSU S
3SU S35        T" U TSU S3SU S35        T" U TSU S3SU S35        T" U TSU S3SU S35        T" U TSU S3SU S3SU S3SU S3/5        T" U TSU S3SU S35        T" U TSU S3SU S35        T" U TSU S3SU S35        M     [        S5       HM  nT" U TSU S3SU S3SU S3SU S	3SU S 3// S!QS"9  T" U TSU S#3SU S$35        T" U TSU S%3SU S&35        MO     U4S' jn[        S( U  5       5      (       a  U" S)S*/U 5        [        S+ U  5       5      (       a
  U" S,/U 5        [        S- U  5       5      (       a
  U" S./U 5        [        S/ U  5       5      (       a  U" S0S1/U 5        [        S2 U  5       5      (       a  U" S3S4/U 5        [        S5 U  5       5      (       a  U" S6S7[        4S8/U 5        [        U R	                  5       5      n0 nU(       GaR  [        S9 U 5       5      (       d  [        S:S;R                  U5       35      eU GH  nUR                  S<5      (       d  M  UR                  S=5      S>   nU S?3nU S@3n[        XV5      n	UR                  SA5      (       aG  U R                  U5      n
U
R                  S>   nXU	'   U R                  U5      XIR                  SBSC5      '   X;   d  M  U R                  U5      R                  5       nUW-  nUnSDnUSE-  U:  a  USE-  nUSE-  nUSE-  U:  a  M  XI==   U-  ss'   XIR                  SBSC5      ==   U-  ss'   GM     [!        U 5      S>:  a&  ["        R%                  SFU R	                  5        35        U(       a)  UR'                  5        VVs0 s H  u  nnSGU 3U_M     nnn0 TEUEnU$ s  snnf )HN   lora_unet_double_blocks__img_attn_projtransformer.transformer_blocks..attn.to_out.0_img_attn_qkv
.attn.to_q
.attn.to_k
.attn.to_v
_img_mlp_0z.ff.net.0.proj
_img_mlp_2z	.ff.net.2_img_mod_linz.norm1.linear_txt_attn_proj.attn.to_add_out_txt_attn_qkv.attn.add_q_proj.attn.add_k_proj.attn.add_v_proj
_txt_mlp_0z.ff_context.net.0.proj
_txt_mlp_2z.ff_context.net.2_txt_mod_linz.norm1_context.linear&   lora_unet_single_blocks__linear1&transformer.single_transformer_blocks.z	.proj_mlp)   r   r   i 0  )r   _linear2	.proj_out_modulation_linz.norm.linearc                    > S H\  nUS:X  a  SOSnU  HI  u  pEnUR                  US9nUR                  US9nUR                  U5      n	U(       a  U" U	5      n	U	T
U'   MK     M^     g )Nlora_Alora_Br  	lora_downlora_up)lora_key)orig_lora_key)formatr8   )assignmentssourcer	  r
  
target_fmt
source_fmt	transform
target_key
source_keyvaluer   s             r   assign_remaining_weightsr_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.assign_remaining_weights%  sv    0/78/C9D5JI!+!2!2H!2!EJ!+!2!2!2!OJ"JJz2E  )% 0).F:& :E 1r   c              3   ,   #    U  H
  nS U;   v   M     g7f)guidance_inNr   r`   s     r   r#   c_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.<genexpr>0       26a}!6   )z<time_text_embed.guidance_embedder.linear_1.{lora_key}.weightz5lora_unet_guidance_in_in_layer.{orig_lora_key}.weightN)z<time_text_embed.guidance_embedder.linear_2.{lora_key}.weightz6lora_unet_guidance_in_out_layer.{orig_lora_key}.weightNc              3   ,   #    U  H
  nS U;   v   M     g7f)img_inNr   r`   s     r   r#   r  A       -fx1}fr  )zx_embedder.{lora_key}.weightz'lora_unet_img_in.{orig_lora_key}.weightNc              3   ,   #    U  H
  nS U;   v   M     g7f)txt_inNr   r`   s     r   r#   r  I  r  r  )z"context_embedder.{lora_key}.weightz'lora_unet_txt_in.{orig_lora_key}.weightNc              3   ,   #    U  H
  nS U;   v   M     g7f)time_inNr   r`   s     r   r#   r  Q  s     .v!yA~vr  )z<time_text_embed.timestep_embedder.linear_1.{lora_key}.weightz1lora_unet_time_in_in_layer.{orig_lora_key}.weightN)z<time_text_embed.timestep_embedder.linear_2.{lora_key}.weightz2lora_unet_time_in_out_layer.{orig_lora_key}.weightNc              3   ,   #    U  H
  nS U;   v   M     g7f)	vector_inNr   r`   s     r   r#   r  b  s     0A{ar  )z8time_text_embed.text_embedder.linear_1.{lora_key}.weightz3lora_unet_vector_in_in_layer.{orig_lora_key}.weightN)z8time_text_embed.text_embedder.linear_2.{lora_key}.weightz4lora_unet_vector_in_out_layer.{orig_lora_key}.weightNc              3   ,   #    U  H
  nS U;   v   M     g7f)final_layerNr   r`   s     r   r#   r  s  r  r  z!norm_out.linear.{lora_key}.weightz?lora_unet_final_layer_adaLN_modulation_1.{orig_lora_key}.weight)zproj_out.{lora_key}.weightz3lora_unet_final_layer_linear.{orig_lora_key}.weightNc              3   B   #    U  H  oR                  S 5      v   M     g7f))lora_telora_te1Nru   r`   s     r   r#   r    s     Un||$;<<n   zIncompatible keys detected: 

 rr   rk   r4   r   rl   rm   rp   rn   ro   r   r   z!Unsupported keys for ai-toolkit: r   )r   r7   r   r5   r6   r   r=   r@   rt   r;   rx   ru   r8   r   rw   ry   r|   r}   warningr   )r   rS   r  remaining_keysr   r"   r   r   r   r   r   r   r   r   r   r   r   r   rK   r   r   r   s                      @r   !_convert_sd_scripts_to_ai_toolkitP_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit  s   rA"*1#^<1!NC	 '*1#];5aS
C5aS
C5aS
C		 #*1#Z81!NC	 #*1#Z81!I>	 #*1#\:1!MB	 #*1#^<1!4DE	 '*1#];5aS8HI5aS8HI5aS8HI		 #*1#Z81!4JK	 #*1#Z81!4EF	 #*1#\:1!4IJ	 L rA&*1#X6<QCzJ<QCzJ<QCzJ<QCyI	 / #*1#X689E	 #*1#_=8<H	' 6		/ 26222$
   -f---$e 	 -f---$k 	 .v...$
   0000$
   26222$ <Y(
 p 
 fkkm,UnUUU #DTYY~E^D_!`aa%||$677IIcN1-	"+O<%.Kv"6!?!O''(ABB"(**S/K#.#4#4Q#7L4?.1NTjjYeNfM"8"86"JK","JJ7<<>E!L0E!&J"H$q.83"a
 A %q.83 "1Z?1!"8"86"JKxWK5 &8 v;?NN>v{{}oNO^k^q^q^st^sGZ{TZ}[M:FB^sMt4F4m4 us   "P c                 	  ^ 0 nS nU  Vs1 s HM  nUR                  S5      (       a  M  UR                  SS5      R                  SS5      R                  SS5      iMO     nn[        S U 5       5      (       d   SU< 35       eS	nU GHj  mTR                  S
5      (       a6  [        TR	                  S
5      S   R	                  S5      S   5      nSU 3nGOTR                  S5      (       a6  [        TR	                  S5      S   R	                  S5      S   5      nSU 3nGO9TR                  S5      (       a  SnM  TR                  S5      (       a  SnGOTR                  S5      (       a  SnOTR                  S5      (       a  SnOTR                  S5      (       a  SnOTR                  S5      (       a#  [        TR	                  S5      S   5      nSU 3nOTR                  S5      (       a#  [        TR	                  S5      S   5      nSU 3nOHTR                  S5      (       a#  [        TR	                  S5      S   5      nS U 3nO[        S!T S"35      eS#T;   a  S$T;   a  US%-  nOyS&T;   a  US'-  nOm[        U4S( jS) 5       5      (       a  TR	                  S#5      S   nUS*U 3-  nO6[        U4S+ jS, 5       5      (       a  TR	                  S#5      S   nUS*U 3-  nU" TXpU5        GMm     U(       a  [        R                  " S-5      n	S.S/S0S1S2.n
S3S4S5.nU H  mTR                  S5      (       d  M  U	R                  T5      nU(       d  M5  [        UR                  S65      5      nS7U 3nS8T;   a&  U
R                  5        H  u  pUT;   d  M  X~-  n  O.   O+S9T;   a%  UR                  5        H  u  pUT;   d  M  X~-  n  O   U" TXpU5        M     S	nU (       a  [        S: U  5       5      nU(       a3  [        U R                  5       5      nU H  mU R                  T5        M     [        U 5      S:  a&  [!        S;[        U R                  5       5       S<35      eUR                  5        VVs0 s H$  u  nnUR                  S=5      (       a  M  S>U 3U_M&     nnnUR                  5        VVs0 s H$  u  nnUR                  S=5      (       d  M  S?U 3U_M&     nnn0 UEUE$ s  snf s  snnf s  snnf )@Nc                 <   U  S3nUR                  U5      nUR                  S   nU  S3nUR                  U5      nU  S3n	UR                  U	5      n
X-  nUnSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  X\-  nX-  nU S3nXSU'   XUR                  SS	5      '   g )
Nr   r   rl   rm   r   r   r   z.lora_A.z.lora_B.)r8   r   rw   )original_keydiffusers_keyrA   rK   down_keyr   	lora_rankup_weight_keyr   	alpha_keyr   r   r   r   diffusers_down_keys                  r   _converti_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers.<locals>._convert  s    &'89H$..2K#))!,I+nO<M"}5I'./INN9-E %EJHq.8+a
A q.8+ &2K!,I$1?.!A1<-.QZ-55j*MNr   r_   r   r   rl   rm   c              3   B   #    U  H  oR                  S 5      v   M     g7f))lora_transformer_rf   Nr)  r`   s     r   r#   j_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers.<locals>.<genexpr>  s     ]_<< BCC_r*  zall_unique_keys=F+lora_transformer_single_transformer_blocks_r.   r   r   single_transformer_blocks.$lora_transformer_transformer_blocks_transformer_blocks.rf   T!lora_transformer_context_embeddercontext_embedder lora_transformer_norm_out_linearnorm_out.linearlora_transformer_proj_outr   lora_transformer_x_embedder
x_embedder:lora_transformer_time_text_embed_guidance_embedder_linear_z)time_text_embed.guidance_embedder.linear_6lora_transformer_time_text_embed_text_embedder_linear_z%time_text_embed.text_embedder.linear_:lora_transformer_time_text_embed_timestep_embedder_linear_z)time_text_embed.timestep_embedder.linear_zHandling for key (z) is not implemented.attn_	_to_out_0r   _to_add_outr   c              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   )r    qkvra   s     r   r#   r<    s     F-Ec-Er&   to_qto_kto_v.attn.c              3   ,   >#    U  H	  oT;   v   M     g 7fr   r   )r    add_qkvra   s     r   r#   r<    s     `5_'A5_r&   )
add_q_proj
add_k_proj
add_v_projz(lora_te1_text_model_encoder_layers_(\d+)z.self_attn.q_projz.self_attn.k_projz.self_attn.v_projz.self_attn.out_proj)q_projk_projv_projout_projz.mlp.fc1z.mlp.fc2)fc1fc2r/   ztext_model.encoder.layers.attnr   c              3   B   #    U  H  oR                  S 5      v   M     g7fr_   Nr)  r`   s     r   r#   r<    s     $TA\\,%?%?r*  z\Expected an empty state dict at this point but its has these keys which couldn't be parsed: r4   ztext_model.transformer.r   )ru   rw   r   r:   r;   NotImplementedErrorr7   r   compilesearchgroupr   r5   r6   r8   r|   r=   )rA   rK   r8  ra   all_unique_keyshas_te_keysrS   r2  	remaininglayer_patternattn_mappingmlp_mappingmatchkey_fragmentsuffixremaining_all_unetr6   r   transformer_state_dictr   s      `                r   (_convert_mixture_state_dict_to_diffusersW_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers  s4   	[8  
<</ dAII)2.667H"MUUV^`bc 	 

 ]_]]]tbrbqas_tt] A||IJJ MNrRXXY\]^_`a"<QC @DEE FGKQQRUVWXYZ"5aS 9k**"ABB 2@AA 19:: *;<< ,Z[[ \]^`ab"KA3 OVWW XYZ\]^"Gs KZ[[ \]^`ab"KA3 O),>qcAV*WXX!|!#!%55M"a'!%77MF-EFFF ! 0 4I!vi[%99M`5_``` ! 0 4I!vi[%99MQ>BU !X JJ'RSM---1	L #-Z@K$||K00%,,Q/A'"<QC @Q;0<0B0B0D,'1,)3M! 1E aZ0;0A0A0C,'1,)3M! 1D
 M~F+ %. #!$$T$T!T
)*Dq!  z?Qnost~  uD  uD  uF  pG  oH  HI  J 
 /=.B.B.D"
.DdaALLYfLg!l1#!.D 	 "
 =K<P<P<Rr<RDAqVWVbVbcpVq/=,a/<Rr:(:M::K
B"
 ss"   S,6S,7S1
S18S7
S7c              3   B   #    U  H  oR                  S 5      v   M     g7f)rc  Nr)  r`   s     r   r#   8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr>2  s     OJqll>::Jr*  rc  rk   zlora_A.weightzlora_up.weightzlora_B.weightc              3      #    U  H9  oR                  S 5      =(       a    SU;   =(       d    SU;   =(       d    SU;   v   M;     g7f)r;  r  r  r   Nr)  r`   s     r   r#   rv  <  s@      lvgh()b{a/?/a9PQ>/aU\`aUablvs   AAdiffusion_model.r_   z!text_encoders.clip_l.transformer.re   c              3   ,   #    U  H
  nS U;   v   M     g7f)position_embeddingNr   r`   s     r   r#   rv  E  s     $S
1%9Q%>
r  rz  zThe `position_embedding` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.zThe state_dict has position_embedding LoRA params and we currently do not support them. Open an issue if you need this supported - https://github.com/huggingface/diffusers/issues/new.c              3   B   #    U  H  oR                  S 5      v   M     g7f) text_encoders.t5xxl.transformer.Nr)  r`   s     r   r#   rv  U  s     ]R\Q%GHHR\r*  ztext_encoders.t5xxlzThe `t5xxl` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.zT5-xxl keys found in the state dict, which are currently unsupported. We will filter them out.Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.r|  c              3   \   #    U  H"  nS U;   =(       a    UR                  S5      v   M$     g7f)diff_br_   re   Nr)  r`   s     r   r#   rv  d  s)     g\fWXAR!,,7Q*RR\frg   .diff_bzThe `diff_b` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.z`diff_b` keys found in the state dict which are currently unsupported. So, we will filter out those keys. Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.c              3   D   #    U  H  nS U;   =(       a    SU;   v   M     g7f).norm.diffNr   r`   s     r   r#   rv  t  s      N:aGqL9W\9:rc   r  zThe `diff` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.zNormalization diff keys found in the state dict which are currently unsupported. So, we will filter out those keys. Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.r  r  r  c              3   ,   #    U  H
  nS U;   v   M     g7f)r   Nr   r`   s     r   r#   rv    s     0Zw!|Zr  r   r  c              3   ,   #    U  H
  nS U;   v   M     g7f)r   Nr   r`   s     r   r#   rv    s     :z! A%zr  z`text_projection` keys found in the `state_dict` which are unexpected. So, we will filter out those keys. Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.r   r   )	r7   r   ru   rw   r   r}   r~   append_custom_replace)rA   r-  rs  has_peft_state_dictra   r   has_mixturehas_position_embeddingzero_status_pe	has_t5xxlzero_status_t5	has_diffbzero_status_diff_bhas_norm_diffzero_status_difflimit_substringsr   r   s                   @@r   %_convert_kohya_flux_lora_to_diffusersr  i  s!   `*AFgRB;N OJOO #((*
*||N+ iAII(/:BBCSUdeghh* 	 

   lv K
 Q[QaQaQcdQcaii 2LA1DQc
d`j`p`p`rs`rX\XYaii CZPRSS`r
s!$$S
$S!S!0=QRNe v ,6+;+;+=_+=41AU]^A^$!$+=J_]R\]]	0=RSNe
 p ,6+;+;+=v+=41Q\\RtEu$!$+=Jvg\fgg	!4Z!K!e
 K
 ,6+;+;+=T+=41RSAS$!$+=JTN:NN2:wGe
 K
 ,6+;+;+=g+=41PQAQ$V]efVf$!$+=Jg'30Z000##G, #((*
*||67 4OA0!3* 	 
 :z:::KKG
 ,6+;+;+=\+=41ARZ[A[$!$+=J\7
CC,Z88K
 es  ` w  U  h
 ]sf   P!'P:P/P";P(P(>P.P.P4!P4P:$P:,P:4Q Q &Q6Qc                    0 n[        U R                  5       5      nS S jnU GHk  nUR                  S5      (       a  [        R                  " SU5      R                  S5      nSU 3nSU;   a  US-  nOYSU;   a  US	-  nOMS
U;   a!  SU;  a  U" U UUSU S3SU S3SU S3/5        O&SU;   a   SU;  a  U" U UUSU S3SU S3SU S3/5        SU;   a  US-  nOSU;   a  US-  nOUR                  S5      (       au  [        R                  " SU5      R                  S5      nSU 3nSU;   a  US-  nO&SU;   a   SU;  a  U" U UUSU S3SU S3SU S3/5        SU;   a  US-  nOSU;   a  US-  nOUnSU;  d  GMX  U R                  U5      X'   GMn     [        U 5      S:  a&  [        S[        U R                  5       5       S35      eU$ )!Nc                 ,   U R                  U5      nU R                  UR                  SS5      5      n[        U5      nUc  UR                  S   U-  /U-  nO[	        U5      UR                  S   :X  d   eU Vs/ s H  oS-   PM	     n	nU Vs/ s H  oS-   PM	     n
nUR                  [        R                  X5      5        UR                  [        U
[        R                  " XdSS95       VVs0 s H  u  pX_M	     snn5        g s  snf s  snf s  snnf )Nz.down.weightz
.up.weightr   r   r   r	   )r8   rw   r|   r   r   rz   r   r   r   r   r;   )r   r   r   r   r   r   r   r   ra   r   r   r   s               r   
handle_qkv9_convert_xlabs_flux_lora_to_diffusers.<locals>.handle_qkv  s    jj)JJw~|LM	 ]
<OOA&*45
BDt9	 2222 8@@x!--x@5=>X++X> 	dmmM?@ 	KYZ[9\(]^(]qt(]^_ A> _s   :DD/D
)zdiffusion_model.double_blocksdouble_blockszdouble_blocks\.(\d+)r/   r   zprocessor.proj_lora1r   zprocessor.proj_lora2r   zprocessor.qkv_lora2upr   r   r   zprocessor.qkv_lora1r   r   r   downr   r   )zdiffusion_model.single_blockssingle_blockszsingle_blocks\.(\d+)r   	proj_lorar  qkv_lorarO  r   z2`old_state_dict` should be at this point but has: r4   r   )	r5   r6   ru   r   rf  rg  r8   r|   r=   )old_state_dictrK   	orig_keysr  old_key	block_numrY   s          r   %_convert_xlabs_flux_lora_to_diffusersr    sk   N^((*+I`* PQQ		"97CII!LI7	{CG%0++'72--&'1d'6I""9)DTU9)DTU9)DTU		 ''1d'6I""9)JO9)JO9)JO		  ++++  RSS		"97CII!LI>ykJGg%;&w&4w+>""@:V@:V@:V		  ++++ G &4&8&8&AN#Q T >QMdSaSfSfShNiMjjklmmr   r"   
substringsreturnc                 (   SSR                  S U 5       5      -   S-   n[        R                  " X 5      nU(       aF  UR                  5       nUS:  a  XS-
     S:X  a  US-
  nOUnU S U R	                  SS5      nXS  nXg-   $ U R	                  SS5      $ )	N(|c              3   N   #    U  H  n[         R                  " U5      v   M     g 7fr   )r   escape)r    r   s     r   r#   "_custom_replace.<locals>.<genexpr>  s     BzRYYs^^zs   #%)r   r/   r4   r   )r@   r   rf  startrw   )r"   r  r   rn  	start_subboundaryleftrights           r   r  r    s     CHHBzBBBSHGIIg#EKKM	q=SQ/36 1}H H9H~%%c3/I|{{3$$r   c                    0 n[        U R                  5       5      nSnSnSnSnS GH0  nU R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  S
U S35      USU S3'   S
U S	3U;   a  U R                  S
U S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   [        S U  5       5      nU(       a  U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   a  U R                  SU S	35      USU S	3'   U R                  SU S35      USU S3'   SU S	3U;   d  GM  U R                  SU S	35      USU S	3'   GM3     [	        U5       GH'  n	SU	 S3n
S GH  nU R                  SU	 SU S35      X SU S3'   SU	 SU S	3U;   a   U R                  SU	 SU S	35      X SU S	3'   U R                  SU	 SU S35      X SU S3'   SU	 SU S	3U;   a   U R                  SU	 SU S	35      X SU S	3'   US :X  a  U R                  SU	 S!U S35      n[
        R                  " U/5      X S"U S3'   [
        R                  " U/5      X S#U S3'   [
        R                  " U/5      X S$U S3'   U R                  SU	 S%U S35      n[
        R                  " U/5      X S&U S3'   [
        R                  " U/5      X S'U S3'   [
        R                  " U/5      X S(U S3'   GO[
        R                  " U R                  SU	 S!U S35      S)S*S+9u  pn[
        R                  " U/5      X S#U S3'   [
        R                  " U/5      X S$U S3'   [
        R                  " U/5      X S"U S3'   [
        R                  " U R                  SU	 S%U S35      S)S*S+9u  nnn[
        R                  " U/5      X S&U S3'   [
        R                  " U/5      X S'U S3'   [
        R                  " U/5      X S(U S3'   SU	 S!U S	3U;   a  [
        R                  " U R                  SU	 S!U S	35      S)S*S+9u  nnn[
        R                  " U/5      X S#U S	3'   [
        R                  " U/5      X S$U S	3'   [
        R                  " U/5      X S"U S	3'   SU	 S%U S	3U;   a  [
        R                  " U R                  SU	 S%U S	35      S)S*S+9u  nnn[
        R                  " U/5      X S&U S	3'   [
        R                  " U/5      X S'U S	3'   [
        R                  " U/5      X S(U S	3'   U R                  SU	 S,U S35      X S-U S3'   SU	 S,U S	3U;   a   U R                  SU	 S,U S	35      X S-U S	3'   U R                  SU	 S.U S35      X S/U S3'   SU	 S.U S	3U;   a   U R                  SU	 S.U S	35      X S/U S	3'   U R                  SU	 S0U S35      X S1U S3'   SU	 S0U S	3U;   a   U R                  SU	 S0U S	35      X S1U S	3'   U R                  SU	 S2U S35      X S3U S3'   SU	 S2U S	3U;   a   U R                  SU	 S2U S	35      X S3U S	3'   U R                  SU	 S4U S35      X S5U S3'   SU	 S4U S	3U;   a   U R                  SU	 S4U S	35      X S5U S	3'   U R                  SU	 S6U S35      X S7U S3'   SU	 S6U S	3U;   d  GM  U R                  SU	 S6U S	35      X S7U S	3'   GM     U R                  SU	 S835      X S93'   U R                  SU	 S:35      X S;3'   U R                  SU	 S<35      X S=3'   U R                  SU	 S>35      X S?3'   GM*     [	        U5       GH  n	S@U	 S3n
S GHX  nU R                  SAU	 SBU S35      X SCU S3'   SAU	 SBU S	3U;   a   U R                  SAU	 SBU S	35      X SCU S	3'   [        XV-  5      nXUUU4nUS :X  Ga7  U R                  SAU	 SDU S35      n[
        R                  " U/5      X S#U S3'   [
        R                  " U/5      X S$U S3'   [
        R                  " U/5      X S"U S3'   [
        R                  " U/5      X SEU S3'   SAU	 SDU S	3U;   a  U R                  SAU	 SDU S	35      n[
        R                  " U/5      X S#U S	3'   [
        R                  " U/5      X S$U S	3'   [
        R                  " U/5      X S"U S	3'   [
        R                  " U/5      X SEU S	3'   GOg[
        R                  " U R                  SAU	 SDU S35      US*S+9u  nnnn [
        R                  " U/5      X S#U S3'   [
        R                  " U/5      X S$U S3'   [
        R                  " U/5      X S"U S3'   [
        R                  " U /5      X SEU S3'   SAU	 SDU S	3U;   a  [
        R                  " U R                  SAU	 SDU S	35      US*S+9u  n!n"n#n$[
        R                  " U!/5      X S#U S	3'   [
        R                  " U"/5      X S$U S	3'   [
        R                  " U#/5      X S"U S	3'   [
        R                  " U$/5      X SEU S	3'   U R                  SAU	 SFU S35      X SGU S3'   SAU	 SFU S	3U;   d  GM8  U R                  SAU	 SFU S	35      X SGU S	3'   GM[     U R                  SAU	 SH35      X S93'   U R                  SAU	 SI35      X S;3'   GM     S H  nU R                  SJU S35      USGU S3'   SJU S	3U;   a  U R                  SJU S	35      USGU S	3'   [        U R                  SKU S35      5      USLU S3'   SKU S	3U;   d  Mv  [        U R                  SKU S	35      5      USLU S	3'   M     [        U 5      S*:  a  [        SMU R                  5       < S35      e[        UR                  5       5       H  n%UR                  U%5      USNU% 3'   M     U$ )ONr   r   r         @r  ztime_in.in_layer..weightz+time_text_embed.timestep_embedder.linear_1..biasztime_in.out_layer.z+time_text_embed.timestep_embedder.linear_2.zvector_in.in_layer.z'time_text_embed.text_embedder.linear_1.zvector_in.out_layer.z'time_text_embed.text_embedder.linear_2.c              3   ,   #    U  H
  nS U;   v   M     g7f)guidanceNr   r`   s     r   r#   >_convert_bfl_flux_control_lora_to_diffusers.<locals>.<genexpr>H  s     H4Gq:?4Gr  zguidance_in.in_layer.z+time_text_embed.guidance_embedder.linear_1.zguidance_in.out_layer.z+time_text_embed.guidance_embedder.linear_2.ztxt_in.zcontext_embedder.zimg_in.zx_embedder.r@  r4   double_blocks..img_mod.lin.norm1.linear..txt_mod.lin.norm1_context.linear.r  .img_attn.qkv.
attn.to_v.
attn.to_q.
attn.to_k..txt_attn.qkv.attn.add_q_proj.attn.add_k_proj.attn.add_v_proj.   r   r	   .img_mlp.0.ff.net.0.proj..img_mlp.2.	ff.net.2..txt_mlp.0.ff_context.net.0.proj..txt_mlp.2.ff_context.net.2..img_attn.proj.attn.to_out.0..txt_attn.proj.attn.to_add_out.z.img_attn.norm.query_norm.scalezattn.norm_q.weightz.img_attn.norm.key_norm.scalezattn.norm_k.weightz.txt_attn.norm.query_norm.scalezattn.norm_added_q.weightz.txt_attn.norm.key_norm.scalezattn.norm_added_k.weightr>  single_blocks..modulation.lin.norm.linear.	.linear1.	proj_mlp.	.linear2.	proj_out.z.norm.query_norm.scalez.norm.key_norm.scalefinal_layer.linear.zfinal_layer.adaLN_modulation.1.znorm_out.linear.W`original_state_dict` should be empty at this point but has original_state_dict.keys()=rc  )r5   r6   r8   r7   r   r   r   r   r:   r;   r   r|   r=   )&original_state_dictconverted_state_dictoriginal_state_dict_keys
num_layersnum_single_layers	inner_dim	mlp_ratior	  has_guidancerS   block_prefixsample_lora_weightcontext_lora_weightsample_qsample_ksample_v	context_q	context_k	context_vsample_q_biassample_k_biassample_v_biascontext_q_biascontext_k_biascontext_v_biasmlp_hidden_dim
split_sizelora_weight	lora_biasqra   r   r   q_biask_biasv_biasmlp_biasr"   s&                                         r   +_convert_bfl_flux_control_lora_to_diffusersr    s   #$7$<$<$>?JII(  ##&7z$IJ 	J8*T[\] xj.2JJ#''*;H:U(KL !#NxjX]!^_
  ##&8
'$JK 	J8*T[\]  z/3KK#''*<XJe(LM !#NxjX]!^_
 ]p\s\s!(73]
FxjPWXY !
%04LL^q^u^u%hZu5_ #J8*TY!Z[ ]p\s\s"8*G4]
FxjPWXY "(515MM^q^u^u&xj6_ #J8*TY!Z[
 H4GHH#''*?z(QR !#NxjX_!`a 'xj6:RR'++.CH:U,ST %'RS[R\\a%bc
 $''*@
'(RS !#NxjX_!`a (z7;SS'++.DXJe,TU %'RS[R\\a%bc
 GZF]F]hZw'G
0
'BC XJe$(@@H[H_H_(5)I #4XJe!DE
 AT@W@WZabjakkrXs@t{8*G<=XJe$(@@BUBYBY\cdlcmmrZsBt ;xj!>?E )J :,QCq1,HTgTkTk =
'BU >xj!PQ  s-z?C[[ViVmVm$QC}XJeDW$~]8*E%RS ]p\s\s =
'B] >1FxjPW!XY  s-z?C[[^q^u^u$QC}XJeD_$~5J8*TY%Z[
 8#%8%<%<~aSP^_g^hho=p%q"UZU^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QR&9&=&=qcQ_`h_iip>q&r#[`[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX 05{{'++nQC~hZW^,_`bcij0,H V[U^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QR27++'++nQC~hZW^,_`bcij3/	9i \a[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WXs.
%@D\\>Ckk'++nQC~hZW\,]^`agh?;}m TYS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPs.
%@D\\AF'++nQC~hZW\,]^`aghB> Z_YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UV ViUlUl ;xj@V >z!QR  s+hZu=AYYWjWnWn$QC{8*EBX$~^H:U%ST QdPgPg ;xj@Q >8*G!LM  s+hZu=AYYReRiRi$QC{8*EBS$~Yxj%NO ^q]t]t ;xj@^ >1GzQX!YZ  s+hZu=AYY_r_v_v$QC{8*EB`$~5KH:UZ%[\ YlXoXo ;xj@Y >1B8*G!TU  s+hZu=AYYZmZqZq$QC{8*EB[$~5FxjPU%VW
 ViUlUl ?8*GDV >z!QR  s/(5AE]]WjWnWn$QCxjFX$~^H:U%ST XkWnWn ?8*GDX >1A(7!ST  s/(5AE]]YlYpYp$QCxjFZ$~5EhZu%UV_ -h EXD[D[QC>?E
~-?@A EXD[D[QC<=E
~-?@A K^JaJaQC>?K
~-EFG K^JaJaQC<=K
~-EFGA J $%3A3a8,HSfSjSj #3H:WET >hZw!OP  s"28*EBF^^UhUlUl$QC'7zGV$~\(5%QR
 !!67N#	>JJ8#155qcS[R\\c6deUZU^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRTYT]T]_j^kTl$~Yxj%PQ#A3iz?C[[ 3 7 7.9U]T^^c8d eIW\W`W`bkalWm(>H:U)STW\W`W`bkalWm(>H:U)STW\W`W`bkalWm(>H:U)STV[V_V_aj`kVl(>8*E)RS${{'++nQCy
RY,Z[]gmn 1a V[U^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRTYT]T]_b^cTd$~Yxj%PQ#A3iz?C[[7<{{+//.9XJV[0\]_iop84FFFH X]W`W`bhaiWj(>H:U)STW\W`W`bhaiWj(>H:U)STW\W`W`bhaiWj(>H:U)STV[V_V_ai`jVk(>8*E)RS QdPgPg 9XJg>Q >8*G!LM  s)H:U;?WWReRiRi$QCy
%@S$~Yxj%NOe -n EXD[D[QC56E
~-?@A EXD[D[QC34E
~-?@A{ &B )>Q>U>U!(73?
y
':; !
%04LL@S@W@W%hZu5A 9XJe!<= FV##&EhZw$WXF
/zAB -XJe<@XXGW#''*I(SX(YZH #3H:U!CD )" !#sXkXpXpXrWttuvww(--/05I5M5Mc5R|C512 1  r   c                    0 n[        U R                  5       5      nSnSnSnSn[        U5       GH  nSU S3nSn	S GH  n
U R                  U	 S	U S
U
 S35      X SU
 S3'   S	U S
U
 S3U;   a"  U R                  U	 S	U S
U
 S35      X SU
 S3'   U R                  U	 S	U SU
 S35      X SU
 S3'   U
S:X  a  U R                  U	 S	U SU
 S35      n[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   U R                  U	 S	U SU
 S35      n[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   GO[        R                  " U R                  U	 S	U SU
 S35      SSS9u  pn[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R                  " U R                  U	 S	U SU
 S35      SSS9u  nnn[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   S	U SU
 S3U;   a  [        R                  " U R                  U	 S	U SU
 S35      SSS9u  nnn[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   S	U SU
 S3U;   a  [        R                  " U R                  U	 S	U SU
 S35      SSS9u  nnn[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   U R                  U	 S	U SU
 S35      X SU
 S3'   U	 S	U SU
 S3U;   a"  U R                  U	 S	U SU
 S35      X SU
 S3'   U R                  U	 S	U SU
 S35      X SU
 S3'   U	 S	U SU
 S3U;   a"  U R                  U	 S	U SU
 S35      X SU
 S3'   U R                  U	 S	U S U
 S35      X S!U
 S3'   U	 S	U S U
 S3U;   a"  U R                  U	 S	U S U
 S35      X S!U
 S3'   U R                  U	 S	U S"U
 S35      X S#U
 S3'   U	 S	U S"U
 S3U;   a"  U R                  U	 S	U S"U
 S35      X S#U
 S3'   U R                  U	 S	U S$U
 S35      X S%U
 S3'   U	 S	U S$U
 S3U;   a"  U R                  U	 S	U S$U
 S35      X S%U
 S3'   U R                  U	 S	U S&U
 S35      X S'U
 S3'   U	 S	U S&U
 S3U;   d  GM  U R                  U	 S	U S&U
 S35      X S'U
 S3'   GM     GM     [        U5       GH  nS(U S3nS GHn  n
U R                  W	 S)U S*U
 S35      X S+U
 S3'   U	 S)U S*U
 S3U;   a"  U R                  U	 S)U S*U
 S35      X S+U
 S3'   [        XV-  5      nXUUU4nU
S:X  Ga;  U R                  U	 S)U S,U
 S35      n[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X S-U
 S3'   U	 S)U S,U
 S3U;   a  U R                  S)U S,U
 S35      n[        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X S-U
 S3'   GOm[        R                  " U R                  U	 S)U S,U
 S35      USS9u  nnnn [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U/5      X SU
 S3'   [        R
                  " U /5      X S-U
 S3'   U	 S)U S,U
 S3U;   a  [        R                  " U R                  U	 S)U S,U
 S35      USS9u  n!n"n#n$[        R
                  " U!/5      X SU
 S3'   [        R
                  " U"/5      X SU
 S3'   [        R
                  " U#/5      X SU
 S3'   [        R
                  " U$/5      X S-U
 S3'   U R                  U	 S)U S.U
 S35      X S/U
 S3'   U	 S)U S.U
 S3U;   d  GML  U R                  U	 S)U S.U
 S35      X S/U
 S3'   GMq     GM     S HM  n
U R                  W	 S0U
 S35      US/U
 S3'   U	 S0U
 S3U;   d  M/  U R                  U	 S0U
 S35      US/U
 S3'   MO     [        U 5      S:  a  [        S1U R                  5       < S35      e[        UR                  5       5       H  n%UR                  U%5      US2U% 3'   M     U$ )3Nr   r   r   r  r@  r4   zbase_model.model.r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r	   r  r  r  r  r  r  r  r  r  r  r  r  r>  r  r  r  r  r  r  r  r  r  rc  )r5   r6   r   r8   r   r   r   r:   r;   r|   r=   )&r  r  r  r  r  r  r  rS   r  original_block_prefixr	  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  ra   r   r   r  r  r  r  r"   s&                                         r   &_convert_fal_kontext_lora_to_diffusersr  I  s   #$7$<$<$>?JII :,QCq1 3,HTgTkTk()s-zQXYU >xj!PQ  s-z?C[[ViVmVm,-^A3mH:UZ[W$~]8*E%RS ]p\s\s()s-zQXY] >1FxjPW!XY
 8#%8%<%<,-^A3nXJV]^&" V[U^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QR&9&=&=,-^A3nXJV]^'# \a[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX 05{{'++01s.QYPZZab 0,H V[U^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QR27++'++01s.QYPZZab 3/	9i \a[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WXs.
%@D\\>Ckk'++/D.E^TUSVVdemdnns,tu?;}m
 TYS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPs.
%@D\\AF'++/D.E^TUSVVdemdnns,tuB>
 Z_YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UV ViUlUl()s+hZwWV >z!QR ((qcXJeTXppWjWnWn,-^A3k(SXYX$~^H:U%ST QdPgPg()s+hZwWQ >8*G!LM ((qcXJeTXppReRiRi,-^A3k(SXYS$~Yxj%NO ^q]t]t()s+hZwW^ >1GzQX!YZ ((qcXJeTXpp_r_v_v,-^A3k(SXY`$~5KH:UZ%[\ YlXoXo()s+hZwWY >1B8*G!TU ((qcXJeTXppZmZqZq,-^A3k(SXY[$~5FxjPU%VW
 ViUlUl()s/(SZ[V >z!QR ((qc
RWX\ttWjWnWn,-^A3ohZW\]X$~^H:U%ST XkWnWn()s/(SZ[X >1A(7!ST ((qc
RWX\ttYlYpYp,-^A3ohZW\]Z$~5EhZu%UVw -	 H $%3A3a8,HSfSjSj()s:J8*T[\T >hZw!OP ((qc9I(SXY]uuUhUlUl,-^A3>NxjX]^V$~\(5%QR
 !!67N#	>JJ8#155,-^A3izQXY V[U^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRTYT]T]_j^kTl$~Yxj%PQ+,N1#YxjPUVZrr 3 7 7.9U]T^^c8d eIW\W`W`bkalWm(>H:U)STW\W`W`bkalWm(>H:U)STW\W`W`bkalWm(>H:U)STV[V_V_aj`kVl(>8*E)RS${{'++/D.E^TUSVV_`h_iip,qr 1a
 V[U^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRTYT]T]_b^cTd$~Yxj%PQ+,N1#YxjPUVZrr7<{{+//3H2IXYWZZcdlcmmr0st"84FFFH
 X]W`W`bhaiWj(>H:U)STW\W`W`bhaiWj(>H:U)STW\W`W`bhaiWj(>H:U)STV[V_V_ai`jVk(>8*E)RS QdPgPg()s)H:WUQ >8*G!LM ((qc8*ERVnnReRiRi,-^A3izQVWS$~Yxj%NOq - &~ )>Q>U>U$%%8
'J?
y
':; $$$7zGKcc@S@W@W())<XJeLA 9XJe!<= ) !#sXkXpXpXrWttuvww(--/05I5M5Mc5R|C512 1  r   c                    [        U R                  5       5       Vs0 s H  oU R                  U5      _M     nnS nS nS nS nS n0 SS_SS	_S
S_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_S S!_S"S#_S$S%_S&S'_S(S)S%S*S+SS,S-S.S/S0S1S2S3.EnUUUUUS4.n	[        UR                  5       5       Ho  n
U
R                  S55      (       a  UR                  U
5      X*[	        S55      S  '   U
R                  S65      (       d  MP  UR                  U
5      X*[	        S65      S  '   Mq     [        UR                  5       5       HE  n
U
S S  nUR                  5        H  u  pUR                  X5      nM     UR                  U
5      X+'   MG     [        UR                  5       5       H+  n
U	R                  5        H  u  pX;  a  M  U" X5        M     M-     [        UR                  5       5       H  n
UR                  U
5      US5U
 3'   M     U$ s  snf )7Nc                     UR                  U 5      nUR                  SSS9u  p4[        R                  " XC/SS9nXQU R	                  SS5      '   g )Nr   r   r	   final_layer.adaLN_modulation.1rD  )r8   r   r   r   rw   )r"   rA   r   r   r   r   s         r   remap_norm_scale_shift_I_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_norm_scale_shift_*  sK    $||A1|-YY~15
Wa3;;?ARSTr   c                 .   S nSU ;   as  UR                  U 5      nUR                  SSS9u  pEnXAU" U R                  SS5      5      '   XQU" U R                  SS5      5      '   XaU" U R                  SS5      5      '   g UR                  U 5      X" U 5      '   g )	Nc                    U R                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  S	S
5      nUR                  SS5      nUR                  SS5      nU$ )Nzindividual_token_refiner.blocksztoken_refiner.refiner_blockszadaLN_modulation.1rD  r  rB  zt_embedder.mlp.0*time_text_embed.timestep_embedder.linear_1zt_embedder.mlp.2*time_text_embed.timestep_embedder.linear_2
c_embedderztime_text_embed.text_embedderr   r   )rw   )r"   rY   s     r   
rename_keyS_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_txt_in_.<locals>.rename_key1  s    kk"CEcdGoo&:<MNGooh0BCGoo&8:fgGoo&8:fgGool4STGooeT2GNr   self_attn_qkvr  r   r	   	attn.to_q	attn.to_k	attn.to_v)r8   r   rw   )r"   rA   r	  r   rQ  rR  rS  s          r   remap_txt_in_?_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_txt_in_0  s    	 c!^^C(F%||A1|5DPTz#++o{"KLMPTz#++o{"KLMPTz#++o{"KLM*4..*=Jz#'r   c                 J   UR                  U 5      nSU ;   a=  X!U R                  SS5      '   X!U R                  SS5      '   X!U R                  SS5      '   g UR                  SSS9u  p4nX1U R                  SS5      '   XAU R                  SS5      '   XQU R                  SS5      '   g )	Nr  img_attn_qkvr  r  r  r  r   r	   r8   rw   r   r"   rA   r   rQ  rR  rS  s         r   remap_img_attn_qkv_E_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_img_attn_qkv_D  s    $s?CIs{{>;?@CIs{{>;?@CIs{{>;?@%||A1|5DCGs{{>;?@CGs{{>;?@CGs{{>;?@r   c                 J   UR                  U 5      nSU ;   a=  X!U R                  SS5      '   X!U R                  SS5      '   X!U R                  SS5      '   g UR                  SSS9u  p4nX1U R                  SS5      '   XAU R                  SS5      '   XQU R                  SS5      '   g )	Nr  txt_attn_qkvzattn.add_q_projzattn.add_k_projzattn.add_v_projr  r   r	   r  r  s         r   remap_txt_attn_qkv_E_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_txt_attn_qkv_P  s    $s?IOs{{>3DEFIOs{{>3DEFIOs{{>3DEF%||A1|5DIMs{{>3DEFIMs{{>3DEFIMs{{>3DEFr   c                    SnSU ;   d  SU ;   a  UR                  U 5      nSU ;   a>  U R                  SS5      R                  S5      nX1U S3'   X1U S	3'   X1U S
3'   X1U S3'   g X"X#R                  S5      SU-  -
  4n[        R
                  " X5SS9u  pgpU R                  SS5      R                  S5      nXaU S3'   XqU S3'   XU S3'   XU S3'   g SU ;   d  SU ;   a  UR                  U 5      n
SU ;   a>  U R                  SS5      R                  S5      nXU S3'   XU S3'   XU S3'   XU S3'   g X"X*R                  S5      SU-  -
  4n[        R
                  " XSS9u  ppU R                  SS5      R                  S5      nXU S3'   XU S3'   XU S3'   XU S3'   g U R                  SS5      nUR                  S S!5      nUR                  S"S#5      nUR                  S$S%5      nUR                  U 5      X'   g )&Nr   zlinear1.lora_A.weightzlinear1.lora_B.weightr  r  single_transformer_blocksz.linear1.lora_A.weightz.attn.to_q.lora_A.weightz.attn.to_k.lora_A.weightz.attn.to_v.lora_A.weightz.proj_mlp.lora_A.weightr   r  r	   z.linear1.lora_B.weightz.attn.to_q.lora_B.weightz.attn.to_k.lora_B.weightz.attn.to_v.lora_B.weightz.proj_mlp.lora_B.weightzlinear1.lora_A.biaszlinear1.lora_B.biasz.linear1.lora_A.biasz.attn.to_q.lora_A.biasz.attn.to_k.lora_A.biasz.attn.to_v.lora_A.biasz.proj_mlp.lora_A.biasz.linear1.lora_B.biasz.attn.to_q.lora_B.biasz.attn.to_k.lora_B.biasz.attn.to_v.lora_B.biasz.proj_mlp.lora_B.biaslinear2r   q_normattn.norm_qk_normattn.norm_k)r8   rw   removesuffixsizer   r;   )r"   rA   hidden_sizelinear1_weightrY   r  r  ra   r   r   linear1_biasr  r  r  r  s                  r    remap_single_transformer_blocks_R_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_single_transformer_blocks_\  s   "c)-D-K'^^C0N3++o7RS``, DRgY&>?@CQgY&>?@CQgY&>?@BPgY&=>?)EXEXYZE[^_bm^mEmn
${{>1Ma++o7RS``, DEgY&>?@CDgY&>?@CDgY&>?@BEgY&=>?"c)-Bc-I%>>#.L3++o7RS``* BNgY&<=>AMgY&<=>AMgY&<=>@LgY&;<=)EVEVWXEY\]`k\kEkl
38;;|]^3_0++o7RS``* BHgY&<=>AGgY&<=>AGgY&<=>@HgY&;<= kk/3NOGooi<Gooh>Gooh>G",.."5Jr   r  rG  ztime_in.mlp.0r  ztime_in.mlp.2r  zguidance_in.mlp.0z*time_text_embed.guidance_embedder.linear_1zguidance_in.mlp.2z*time_text_embed.guidance_embedder.linear_2zvector_in.in_layerz&time_text_embed.text_embedder.linear_1zvector_in.out_layerz&time_text_embed.text_embedder.linear_2r  r   img_attn_q_normr  img_attn_k_normr!  img_attn_projzattn.to_out.0txt_attn_q_normzattn.norm_added_qtxt_attn_k_normzattn.norm_added_ktxt_attn_projzattn.to_add_outzimg_mod.linearznorm1.linear	img_norm1z
norm1.norm	img_norm2norm2r   znorm1_context.linearnorm2_context
ff_contextznorm.linearz	norm.normznorm_out.normr   
net.0.projnet.2r   )img_mlpztxt_mod.linear	txt_norm1	txt_norm2txt_mlpself_attn_projzmodulation.linearpre_normzfinal_layer.norm_finalzfinal_layer.linearr^  r_  input_embedder)r  r  r  r  r  rc  rx  )r5   r6   r8   ru   r|   r   rw   )r  ra   r  r  r  r  r  r'  TRANSFORMER_KEYS_RENAME_DICTTRANSFORMER_SPECIAL_KEYS_REMAPr"   rY   replace_keyr	  special_keyhandler_fn_inplaces                   r   (_convert_hunyuan_video_lora_to_diffusersrB  '  s   CGH[H`H`HbCcdCca266q99Ccdb>(
H
N26h$,$E$ 	E$ 	I	$
 	I$ 	F$ 	G$ 	-$ 	=$ 	=$ 	$ 	.$ 	.$ 	*$ 	.$  	\!$" 	W#$$ 0!$)*"1(#=$ D  ++9*A&" (--/0>>.))?S?W?WX[?\ S%8%:!;<>>,--CWC[C[\_C` S);%<%>!?@	 1 (--/0a&'C'I'I'K#Kook>G (L(<(@(@(E%	 1 (--/0/M/S/S/U+K%s9 0V 1 (--/05I5M5Mc5R|C512 1  W es   Hc                   ^ ^ T R                  5        VVs0 s H  u  pU[        S5      S  U_M     snnm 0 mS nUU 4S jnSnU" T R                  5       U5      n[        U5       H  nU" SUSS9  M     SnU" T R                  5       U5      n	[        U	5       H  nU" S	US
S9  M     Sn
U" T R                  5       U
5      n[        U5       H  nU" SUSS9  M     [        T 5      S:  a  [	        ST R                  5       < 35      e[        TR                  5       5       H  nTR                  U5      TSU 3'   M     T$ s  snnf )Nrx  c                     [        5       nU  HK  n[        R                  " X5      nU(       d  M"  UR                  [	        UR                  S5      5      5        MM     [        U5      $ )Nr/   )r9   r   rf  r<   r:   rg  r|   )r6   r   layersr"   rn  s        r   get_num_layersH_convert_non_diffusers_lumina2_lora_to_diffusers.<locals>.get_num_layers  sK    CIIg+Eu

3u{{1~./  6{r   c           	        > T	R                  U  SU S35      nT	R                  U  SU S35      nS H  nUTU  SU SU S3'   M     [        / SQ[        R                  " U/ SQSS	95       H  u  pVUTU  SU SU S
3'   M     T	R                  U  SU S35      TU  SU S3'   T	R                  U  SU S35      TU  SU S3'   [	        SS5       HO  nT	R                  U  SU SU S35      TU  SU SU S3'   T	R                  U  SU SU S
35      TU  SU SU S
3'   MQ     U(       aA  T	R                  U  SU S35      TU  SU S3'   T	R                  U  SU S35      TU  SU S3'   g g )Nr4   z.attention.qkv.lora_A.weightz.attention.qkv.lora_B.weightrP  rT  r   )i 	     rI  r   r	   r   z.attention.out.lora_A.weightz.attn.to_out.0.lora_A.weightz.attention.out.lora_B.weightz.attn.to_out.0.lora_B.weightr/      z.feed_forward.wz.feed_forward.linear_z!.adaLN_modulation.1.lora_A.weightz.norm1.linear.lora_A.weightz!.adaLN_modulation.1.lora_B.weightz.norm1.linear.lora_B.weight)r8   r   r   r;   r   )
r   indexconvert_normr  r  attn_keyr   rP   r  rA   s
           r   process_blockG_convert_non_diffusers_lumina2_lora_to_diffusers.<locals>.process_block  s,   NNfXQug5Q#RS	..F81UG3O!PQ0HV_ F81UG6(>!RS 1 #$<ekk'Scij>k lHV\ F81UG6(>!RS !m Q[P^P^haw:;Q
xq/KLM Q[P^P^haw:;Q
xq/KLM
 1a[Eblbpbp(!E7/%Gc F81UG3H~!^_ cmbpbp(!E7/%Gc F81UG3H~!^_	 ! S]SaSa(!E7"CDT F81UG3N!OP T^SaSa(!E7"CDT F81UG3N!OP	 r   znoise_refiner\.(\d+)\.noise_refinerT)rL  zcontext_refiner\.(\d+)\.context_refinerFzlayers\.(\d+)\.rE  r   E`state_dict` should be empty at this point but has state_dict.keys()=rc  )r   r|   r6   r   r=   r5   r8   )rA   ra   r   rF  rN  noise_refiner_patternnum_noise_refiner_layersrS   context_refiner_patternnum_context_refiner_layerscore_transformer_patternnum_core_transformer_layersr"   r  s   `            @r   0_convert_non_diffusers_lumina2_lora_to_diffusersrY    sb   >H>N>N>PQ>Pda!C*+-.1>PQJ D 6-joo.?AVW+,oqt< - :!/
0ACZ![-.'? /  2"01BD\"]./h5 0 :azO`Nbcdd(--/05I5M5Mc5R|C512 1  E Rs   D?c           
      *  ^$^%^& 0 nU R                  5        VVs0 s H  u  p#U[        S5      S  U_M     snnm%T% Vs1 s H7  o"R                  S5      (       d  M  [        UR	                  S5      S   5      iM9     nn[        U5      n[        U5      n[        S T% 5       5      =(       a    [        S T% 5       5      n[        S T% 5       5      (       a  SOS	n[        S
 T% 5       5      (       a  SOSn	[        S T% 5       5      n
U%4S jn[        T%R                  5       5       Hk  nUR                  S5      (       a0  SU;   a*  T%R                  U5        [        R                  SU S35        SU;   d  MQ  U
(       a  MZ  T%R                  U5        Mm     [        XVS-   5       GH  n[        / SQ/ SQ5       H  u  pSU SU S3nUT%;   nSU SU SU S3nSU SU S3nSU SU SU	 S3nSU SU S3nU(       a?  T%R                  U5      nT%R                  U5      nU" UU5      u  nnUU-  UU'   UU-  UU'   O4UT%;   a  T%R                  U5      UU'   UT%;   a  T%R                  U5      UU'   SU SU S3nSU SU S3nUT%;   d  M  T%R                  U5      UU'   M     [        / SQ/ SQ5       H  u  pSU SU S3nUT%;   nSU SU SU S3nSU SU S3nSU SU SU	 S3nSU SU S3nUT%;   a  T%R                  U5      nUUU'   UT%;   a  T%R                  U5      nUUU'   U(       a&  U" WU5      u  nnUU==   U-  ss'   UU==   U-  ss'   SU SU S3nSU SU S3nUT%;   d  M  T%R                  U5      UU'   M     U(       a  [        S S!/S"S#/5       H  u  pSU SU S3nUT%;   nSU SU SU S3nSU SU S3nSU SU SU	 S3nSU SU S3nUT%;   a  T%R                  U5      nUUU'   UT%;   a  T%R                  U5      nUUU'   U(       a&  U" WU5      u  nnUU==   U-  ss'   UU==   U-  ss'   SU SU S3nSU SU S3nUT%;   d  M  T%R                  U5      UU'   M     [        S$S%/S&S'/5       H  u  pSU SU S3nUT%;   nSU SU SU S3nSU S(U S3nSU SU SU	 S3nSU S(U S3nUT%;   a  T%R                  U5      nUUU'   UT%;   a  T%R                  U5      nUUU'   U(       a&  U" WU5      u  nnUU==   U-  ss'   UU==   U-  ss'   SU SU S3nSU S(U S3nUT%;   d  M  T%R                  U5      UU'   M     GM     T%(       GaM  [        S) T% 5       5      (       a^  S*U S3nS+nUT%;   a  T%R                  U5      UU'   S*U	 S3nS,nUT%;   a  T%R                  U5      UU'   S-T%;   a  T%R                  S-5      US.'   [        S/ U  5       5      (       aJ  T%R                  S0U S35      US1'   T%R                  S0U	 S35      US2'   S3T%;   a  T%R                  S35      US4'   S5 H  m&[        U&4S6 jT% 5       5      (       d  M  S7 H  m$T$S8:X  a  SOS9nT&S::X  a  S;OS<n[        U$U&4S= jT% 5       5      (       aF  T%R                  T& ST$ SU S35      UU S>U S3'   T%R                  T& ST$ SU	 S35      UU S>U S3'   T& ST$ S3T%;   d  M  T%R                  T& ST$ S35      UU S>U S3'   M     M     S? H  u  nnU SU S3nS@U S3nUT%;   a  T%R                  U5      UU'   U SU	 S3nS@U S3nUT%;   d  MD  T%R                  U5      UU'   UR!                  SU	 S35      S-   n U T%;   d  Mx  UR!                  S5      SA-   n!T%R                  U 5      UU!'   M     [        T%5      S8:  a  [#        SB T% 5       5      n"U"(       a\  T% Vs1 s H  o"R                  SC5      (       d  M  UiM     n#n[#        SD U# 5       5      (       d  [$        e[        R'                  SE5        O[%        SFT%R                  5       < 35      e[        UR                  5       5       H  nUR                  U5      USGU 3'   M     U$ s  snnf s  snf s  snf )HNrx  blocks.r4   r/   c              3   ,   #    U  H
  nS U;   v   M     g7fk_imgNr   r`   s     r   r#   ?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>%       @,?qgl,?r  c              3   ,   #    U  H
  nS U;   v   M     g7fv_imgNr   r`   s     r   r#   r_  %       Hs_rZ[TU_rr  c              3   ,   #    U  H
  nS U;   v   M     g7f)r  Nr   r`   s     r   r#   r_  &  s     #O;NaHM;Nr  r  r  c              3   ,   #    U  H
  nS U;   v   M     g7f)r  Nr   r`   s     r   r#   r_  '  s     !M9LA(a-9Lr  r  r  c              3   r   #    U  H-  oR                  S 5      =(       a    UR                  S5      v   M/     g7f)time_projectionr  N)ru   rt   r`   s     r   r#   r_  (  s-      %K^a&'AAJJy,AAK^s   57c                    > U R                   S   nTR                  U5      R                  5       nX2-  nUnSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  XV4$ Nr   r   r   r   r8   ry   )r   r6  r   r   r   r   r   r  s          r   get_alpha_scalesF_convert_non_diffusers_wan_lora_to_diffusers.<locals>.get_alpha_scales,  st      ##''	2779
1nx'!OJMH 1nx' ##r   )r  r  normz	Removing zG key from the state dict as it is a norm diff key. This is unsupported.rh  r  ra   r   orQ  rR  rS  zto_out.0z.self_attn.rm   r  .attn1.r   r   r  z.lora_B.biasz.cross_attn..attn2.r^  rc  rX  rY  zffn.0zffn.2r4  r5  .ffn.c              3   ,   #    U  H
  nS U;   v   M     g7f)rh  Nr   r`   s     r   r#   r_    s     C/B! A%/Br  ztime_projection.1.z*condition_embedder.time_proj.lora_A.weightz*condition_embedder.time_proj.lora_B.weightztime_projection.1.diff_bz(condition_embedder.time_proj.lora_B.biasc              3   ,   #    U  H
  nS U;   v   M     g7f)z	head.headNr   r`   s     r   r#   r_    s     4A{ar  z
head.head.zproj_out.lora_A.weightzproj_out.lora_B.weightzhead.head.diff_bzproj_out.lora_B.bias)text_embeddingtime_embeddingc              3   .   >#    U  H
  nTU;   v   M     g 7fr   r   )r    ra   	text_times     r   r#   r_    s     ?+>a9>+>s   )r   r   r   r   rw  z condition_embedder.text_embedderz condition_embedder.time_embedderc              3   8   >#    U  H  nT S T 3U;   v   M     g7fr4   Nr   )r    ra   b_nrz  s     r   r#   r_    s#     R>Qi[#/14>Qs   z.linear_))zff.net.0.projzimg_emb.proj.1)zff.net.2zimg_emb.proj.3z"condition_embedder.image_embedder.r  c              3   ,   #    U  H
  nS U;   v   M     g7f)r  Nr   r`   s     r   r#   r_    s     =)<A7a<)<r  r  c              3   ,   #    U  H
  nS U;  v   M     g7f)loraNr   r`   s     r   r#   r_    s     :	1vQ	r  zThe remaining `state_dict` contains `diff` keys which we do not handle yet. If you see performance issues, please file an issue: https://github.com/huggingface/diffusers//issues/newN`state_dict` should be empty at this point but has original_state_dict.keys()=rc  )r   r|   ru   r:   r;   minmaxr7   r5   r6   rt   r8   r}   debugr   r   r"  r   r=   r~   )'rA   r  ra   r   block_numbers	min_block	max_blockis_i2v_loralora_down_keylora_up_keyhas_time_projection_weightrl  r"   rS   rp  cr6  	has_alphaoriginal_key_Aconverted_key_Aoriginal_key_Bconverted_key_Br   r   r   r   r1  converted_keydiffusers_b_nr   img_ours
img_theirsbias_key_theirsbias_keydiff	diff_keysr}  r  rz  s'                                       @@@r   ,_convert_non_diffusers_wan_lora_to_diffusersr    s   GQGWGWGYZGYtq1S!3467:GYZ3Fb3Fa,,W`Ja)Sa)3FMbM"IM"I@,?@@sSHs_rHsEsK ##O;N#O O OHU`M!!M9L!MMM(S\K!$ %K^% "	$ ',,./<<,--&C-  ##C(LL9SE)pqr#,F,F  ##C( 0  9!m,,.RSDA!!Ks&9I!%88I&qcQCqwON 's'!NCO&qcQCqWMN 's'!NCO155nE/33NC	'7Y'O$
H8Cj8P$_58AH8L$_5 "%88<O<S<STb<c(9!%88<O<S<STb<c(9$QC{1#W=L%aSs,?M226I6M6Ml6[$]33 T8 ,.RSDA!!L6:I!%88I&qcaS-PN 's'!NCO&qcaS+gNN 's'!NCO!44155nE8C$_5!44/33NC	8A$_5'7Y'O$
H$_5C5$_5A5$QC|A3g>L%aSs,?M226I6M6Ml6[$]3/ T2 Wg.|0LM%aSQCv>	%)<<	#*1#\!Am_G!T$+A3gaS"G#*1#\!Ak]'!R$+A3gaS"G!%88"5"9"9."IK<G(9!%88 3 7 7 GI<E(9+;K+S(J(9ZG9(9XE9!(<s'B")!GA3l C#66:M:Q:QR^:_(7/ N4 '*\7,CDDA!!AaS/I!%88I&qc1#Q}oWEN 's%s.AO&qc1#Q{m7CN 's%s.AO!44155nE8C$_5!44/33NC	8A$_5'7Y'O$
H$_5C5$_5A5$QCq73L%aSaS=M226I6M6Ml6[$]3/ Ee -X C/BCCC/gFLHM226I6M6Ml6[$]3/}GDLHM226I6M6Ml6[$]3)-@@SfSjSj.T$%OP 4444=P=T=T]O73> !9: >Q=T=TWabmannuUv=w !9:!%88?R?V?VWi?j$%;<=I?+>???!C),AqM %(88 ;? #
 R>QRRR/33yk3%qW^4_` -/?xVd-ef 033yk3%qU\4]^ -/?xVd-ef $AcU'26II/33yk3%w4OP -/?xVb-cd " >*%
 Hj )\=/AL@
.YM226I6M6Ml6[$]3(\;-w?L@
.YM226I6M6Ml6[$]3".";";a}G<T"UXa"a"&99,99)DwNH5H5L5L_5](2!%
$ !#=)<==$7O$7q::g;N$7IO:	:::  KKG
 nSfSkSkSmRopqq(--/05I5M5Mc5R|C512 1  I [bj Ps   bb!bb4bc           	        ^ 0 nU R                  5        VVs0 s H  u  p#U[        S5      S  U_M     snnm[        T Vs1 s H(  o"R                  S5      S   R                  S5      S   iM*     sn5      n[        S T 5       5      =(       a    [        S T 5       5      nU4S jn[	        U5       GH  n[        / S	Q/ S
Q5       Hb  u  pTR                  SU SU S35      n
TR                  SU SU S35      nU" U
SU SU 35      u  pX-  USU SU	 S3'   X-  USU SU	 S3'   Md     [        / S	Q/ S
Q5       Hb  u  pTR                  SU SU S35      n
TR                  SU SU S35      nU" U
SU SU 35      u  pX-  USU SU	 S3'   X-  USU SU	 S3'   Md     U(       av  [        SS/SS/5       Hb  u  pTR                  SU SU S35      n
TR                  SU SU S35      nU" U
SU SU 35      u  pX-  USU SU	 S3'   X-  USU SU	 S3'   Md     [        SS/SS/5       Hb  u  pTR                  SU SU S35      n
TR                  SU SU S35      nU" U
SU SU 35      u  pX-  USU SU	 S3'   X-  USU SU	 S3'   Md     GM     [        T5      S:  a  [        STR                  5       < 35      e[        UR                  5       5       H  nUR                  U5      USU 3'   M     U$ s  snnf s  snf )Nr_   blocks_r/   r   r   c              3   ,   #    U  H
  nS U;   v   M     g7fr]  r   r`   s     r   r#   8_convert_musubi_wan_lora_to_diffusers.<locals>.<genexpr>  r`  r  c              3   ,   #    U  H
  nS U;   v   M     g7frb  r   r`   s     r   r#   r    rd  r  c                    > U R                   S   nTR                  US-   5      R                  5       nX2-  nUnSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  XV4$ )Nr   rm   r   r   rk  )r   r"   r   r   r   r   r   r  s          r   rl  ?_convert_musubi_wan_lora_to_diffusers.<locals>.get_alpha_scales  sx      ##''h7<<>
1nx'!OJMH 1nx' ##r   ro  rq  _self_attn_r   rl   r[  rr  r   r   _cross_attn_rs  r^  rc  rX  rY  ffn_0ffn_2r4  r5  rt  r  rc  )
r   r|   r;   r7   r   r   r8   r=   r6   r5   )rA   r  ra   r   
num_blocksr  rl  rS   rp  r  r   r   r   r   r"   r  s                  @r   %_convert_musubi_wan_lora_to_diffusersr    s   AKAQAQASTAS1S.0114ASTCVWCVaggi(+11#6q9CVWXJ@,?@@sSHs_rHsEsK	$ :,.RSDA-11GA3k!L]2^_K+//'!Ks/0Z[I#3K71#[YZX[A\#] JJUJb 71#WQC~!FGJSJ^ 71#WQC~!FG T ,.RSDA-11GA3l1#M^2_`K+//'!L?0[\I#3K71#\Z[Y\A]#^ JJUJb 71#WQC~!FGJSJ^ 71#WQC~!FG T Wg.|0LM155s,qcQb6cd/33gaSQC4_`	'7wqcQ]^_]`Ea'b$
NYNf$wqc>%JKNWNb$wqc>%JK N '*\7,CDDA-11GA3asBS2TUK+//'!AaS0PQI#3K71#QqcAR#S JHSH` 71#U1#^!DEHQH\ 71#U1#^!DE E5 B !#jObOgOgOiNklmm(--/05I5M5Mc5R|C512 1  o UWs
   K>/Lc                 (  ^ [        U4S jU  5       5      (       d  [        S5      eU R                  5        VVs0 s H  u  p#UR                  T S35      U_M     nnnUR                  5        VVs0 s H  u  p#SU 3U_M     nnnU$ s  snnf s  snnf )Nc              3   D   >#    U  H  oR                  T5      v   M     g 7fr   r)  r    ra   non_diffusers_prefixs     r   r#   C_convert_non_diffusers_hidream_lora_to_diffusers.<locals>.<genexpr>D  s     F:a||011:s    z$Invalid LoRA state dict for HiDream.r4   rc  r   r=   r   removeprefixrA   r  ra   r   r  s    `   r   0_convert_non_diffusers_hidream_lora_to_diffusersr  C  s    F:FFF?@@V`VfVfVhiVhdaANN.B-C1+EFIVhi>R>X>X>Z[>Zdal1#.1>Z[ j[   !B1Bc                 (  ^ [        U4S jU  5       5      (       d  [        S5      eU R                  5        VVs0 s H  u  p#UR                  T S35      U_M     nnnUR                  5        VVs0 s H  u  p#SU 3U_M     nnnU$ s  snnf s  snnf )Nc              3   J   >#    U  H  oR                  T S 35      v   M     g7fr|  r)  r  s     r   r#   @_convert_non_diffusers_ltxv_lora_to_diffusers.<locals>.<genexpr>L  s$     LA||34A677s    #z&Invalid LoRA state dict for LTX-Video.r4   rc  r  r  s    `   r   -_convert_non_diffusers_ltxv_lora_to_diffusersr  K  s    LLLLABBV`VfVfVhiVhdaANN.B-C1+EFIVhi>R>X>X>Z[>Zdal1#.1>Z[ j[r  c                   ^  [        S T  5       5      nU(       ar  T R                  5        VVs0 s H  u  p#UR                  S5      U_M     snnm S[        S[        4S jnT R                  5        VVs0 s H  u  p#U" U5      U_M     snnm 0 n[	        T R                  5       5      nSnSnU 4S jn	U H  nUR                  U5      (       d  M  UR                  US	5      n
UR                  Xx5      R                  US
5      nUR                  US5      nT R                  U5      nT R                  UR                  Xx5      5      nU	" X5      u  nnX-  XZ'   UU-  X['   M     [        T 5      S:  a  [        ST R                  5       < 35      eUR                  5        VVs0 s H  u  p#SU 3U_M     nnnU$ s  snnf s  snnf s  snnf )Nc              3   B   #    U  H  oR                  S 5      v   M     g7frb  r)  r`   s     r   r#   @_convert_non_diffusers_qwen_lora_to_diffusers.<locals>.<genexpr>T  s     GJq\22Jr*  r_   r"   r  c           	      *   SnSU ;   a  U R                  SS5      u  p#OU Sp2U S3nU[        U5      S  nSU;   a  UR                  SS5      u  pgSU-   nOUSpv1 Skn0 n	U H5  n
U	R                  [        U
5      [	        5       5      R                  U
5        M7     UR                  S5      n/ nSn[        U	R                  5       SS	9nU[        U5      :  a  S
nU H\  nUU-   [        U5      ::  d  M  [        XUU-    5      U	U   ;   d  M0  UR                  SR                  XUU-    5      5        UU-  nSn  O   U(       d  UR                  X   5        US-  nU[        U5      :  a  M  SR                  U5      nU SU U 3nUU(       a  SU-   -   $ S-   $ )Nr   r4   r/   r   r   >   tora   r  r  r  r   r  outr<   ra   r<   r  r<   r   imgr   r  modtxtr   r  r  r  r<   r  r<   ra   projr<   r  r  r<   r   r  r   T)reverseF)rsplitr|   r;   
setdefaultr9   r<   sortedr6   tupler  r@   )r"   r   baserp  r  restheadtail	protectedprot_by_lenngpartsmergedrS   lengths_descmatchedLhead_convertedconverted_bases                      r   convert_keyB_convert_non_diffusers_qwen_lora_to_diffusers.<locals>.convert_keyX  s   )Fcz"zz#q1f"BfhaLEE
%Dd{!ZZQ/
Tz!2dI( K&&s2w6::2>   JJsOEFA!+"2"2"4dCLc%j.%A1uE
*uUq1u5E/F+VW./XchhuQ/?&@AQ"& & MM%(+FA c%j. !XXf-N &xq(8?N!vcFlFF2FFr   r   rl   c                    > U R                   S   nTR                  U5      R                  5       nX2-  nUnSnUS-  U:  a  US-  nUS-  nUS-  U:  a  M  XV4$ rj  rk  )r   r6  r   r   r   r   r   rA   s          r   rl  G_convert_non_diffusers_qwen_lora_to_diffusers.<locals>.get_alpha_scales  sr      #y)..0
1nx'!OJMH 1nx' ##r   r   r   rm   r   rR  rc  )r7   r   r  r?   r5   r6   rt   rw   r8   r|   r=   )rA   has_lora_unetra   r   r  r  rE   r3  up_keyrl  r7  diffusers_up_keyr6  r   r   r   r   s   `                r   -_convert_non_diffusers_qwen_lora_to_diffusersr  S  s   GJGGMBLBRBRBTUBT$!ann\2A5BTU
<	GS <	GS <	G| 5?4D4D4FG4FDAk!na'4FG
JOO%&H"HF	$ ::h!"85E!F yy:BB6K[\		(H5I$..+K"qyy'BCI#3K#K J7B7O 45>5I 2  :azO`Nbcdd>R>X>X>Z[>Zdal1#.1>Z[E V@ HB \s   F74F= G)r      )unettext_encoder)diffusion_model)r   typingr   r   utilsr   r   r   
get_logger__name__r}   r   r[   r   rv   rx   r{   r  r  r?   r  r  r  rB  rY  r  r  r  r  r  r   r   r   <module>r     s    
   A A 
		H	%vri*X5p8 p9lfR% %$s) % %(j Z	[ |l ^D Nf R: z  e r   