+hi SSKJrJrJrJrJr SSKrSSKrSSK J r SSK J r J r SSKJrJrJr SSKJrJrJrJrJr SSKJr SSKJr S S KJrJr S S KJ r S S K!J"r"J#r#J$r$J%r% S S K&J'r' S SK(J)r) S SK*J+r+J,r,J-r- SSK.J/r/J0r0 \Rb"\25r3"SS\ Rh5r5"SS\ Rh5r6"SS\ Rh5r7"SS\ Rh5r8"SS\ Rh5r9\"SS\ Rh55r:\"SS\ Rh55r;"S S!\)\ \\\\ \5 r[TU]5 Ub[X!5UlOSUlUS:Xa[R "USSS9UlgUS:Xa[USSS9Ulg[SUS35e) N layer_normFư>elementwise_affineepsfp32_layer_norm)r/biasUnsupported `norm_type` (@) provided. Supported ones are: 'layer_norm', 'fp32_layer_norm'.) super__init__rembnn LayerNormnormr ValueError)selfr)r* norm_typer2 __class__s j/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/models/transformers/transformer_chroma.pyr6%ChromaAdaLayerNormZeroPruned.__init__0sx   %6~UDHDH  $ ]uRVWDI + +%mTYZDI+I;6vw xtimestep class_labels hidden_dtyper7returncURbURX#US9nURSS5RSSS9upgppURU5SUSS2S4--USS2S4-nXXU 4$)N)rEr"rdim)r7flattenchunkr:) r<rBrCrDrEr7 shift_msa scale_msagate_msa shift_mlp scale_mlpgate_mlps r?forward$ChromaAdaLayerNormZeroPruned.forward@s 88 ((8 (MCILUVXYIZI`I`abhiI`IjF h9 IIaLA !T' 22 3i46H HI(::rA)r7r:)Nr,T)NNNN)__name__ __module__ __qualname____firstlineno____doc__intrr6torchTensor LongTensordtyperrS__static_attributes__ __classcell__r>s@r?r&r&'sc8C=&,037.2&* ; << ;5<<( ;u//0 ; u{{+ ; ell # ; u||U\\5<<u||S T ; ;rAr&c ^\rSrSrSrS S\4U4SjjjrS S\RS\ \RS\ \R\R\R\R\R44Sjjr S r U=r $) "ChromaAdaLayerNormZeroSinglePrunedOr(r)c>[TU]5 US:Xa[R"USSS9Ulg[ SUS35e)Nr,Fr-r.r3r4)r5r6r8r9r:r;)r<r)r=r2r>s r?r6+ChromaAdaLayerNormZeroSinglePruned.__init__XsE   $ ]uRVWDI+I;6vw rArBr7rFcURSS5RSSS9up4nURU5SUSS2S4--USS2S4-nX4$)Nr"rrrI)rKrLr:)r<rBr7rMrNrOs r?rS*ChromaAdaLayerNormZeroSinglePruned.forwardbs_ *-Q):)@)@)@)J& h IIaLA !T' 22 3i46H H{rAr:)r,TN)rUrVrWrXrYrZr6r[r\rrrSr_r`ras@r?rcrcOsuc'+ <<ell # u||U\\5<<u||S T rArcc^\rSrSrSrS S\S\4U4SjjjrS\RS\RS\R4S jr S r U=r $) "ChromaAdaLayerNormContinuousPrunedlaq Adaptive normalization layer with a norm layer (layer_norm or rms_norm). Args: embedding_dim (`int`): Embedding dimension to use during projection. conditioning_embedding_dim (`int`): Dimension of the input condition. elementwise_affine (`bool`, defaults to `True`): Boolean flag to denote if affine transformation should be applied. eps (`float`, defaults to 1e-5): Epsilon factor. bias (`bias`, defaults to `True`): Boolean flag to denote if bias should be use. norm_type (`str`, defaults to `"layer_norm"`): Normalization layer to use. Values supported: "layer_norm", "rms_norm". r)conditioning_embedding_dimc>[TU]5 US:Xa[R"XX55UlgUS:Xa[ XU5Ulg[ SU35e)Nr,rms_normzunknown norm_type )r5r6r8r9r:r!r;)r<r)rnr/r0r2r=r>s r?r6+ChromaAdaLayerNormContinuousPruned.__init__{sU   $ ]9KRDI * $ 4FGDI1)=> >rArBr7rFc[R"URSS5RUR5SSS9up4UR U5SU-SS2SSS24-USS2SSS24-nU$)Nr"rrI)r[rLrKtor^r:)r<rBr7shiftscales r?rS*ChromaAdaLayerNormContinuousPruned.forwardsf{{3;;q!#4#7#7#@!K  IIaLAIq$z2 2U1dA:5F FrAri)Tgh㈵>Tr,) rUrVrWrXrYrZr6r[r\rSr_r`ras@r?rlrlls] .  ??%(??,ELLU\\rArlcn^\rSrSrS\S\4U4SjjrS\RS\R4SjrSr U=r $) (ChromaCombinedTimestepTextProjEmbeddings num_channelsout_dimc >[TU]5 [USSS9Ul[USSS9UlUR S[ [R"U5S-SU-SSS9SS 9 g) NTr)rzflip_sin_to_cosdownscale_freq_shiftmod_projr)r}r~F) persistent) r5r6r time_proj guidance_projregister_bufferrr[arange)r<rzr{r>s r?r61ChromaCombinedTimestepTextProjEmbeddings.__init__ss " dijk&LRVmno   " W%,a,.>PTkl   rArCrFcURRSnURSnURU5RURS9nUR [ R"S/U-55RURURS9nURRURURS9RUSS5n[ R"XE/SS9RS5RSUS5n[ R"Xv/SS9nURUR5$)Nr)r^)r^devicer"rI) rshaperrsr^rr[tensorrrepeatcat unsqueeze) r<rCmod_index_length batch_sizetimesteps_projrrtimestep_guidance input_vecs r?rS0ChromaCombinedTimestepTextProjEmbeddings.forwards==..q1^^A& 1448>>4J**5<<j8H+IJMM..N ==##.*>*>~G\G\#]ddeoqrtuv II~51 = G G J Q QRSUegh i II0;D ||HNN++rA)rr) rUrVrWrXrZr6r[r\rSr_r`ras@r?rxrxs6  S  3  , ,,,rArxc F^\rSrSrS S\S\S\S\4U4SjjjrSrSrU=r$) ChromaApproximatorin_dimr{ hidden_dimn_layersc >[TU]5 [R"XSS9Ul[R "[ U5Vs/sH n[X3SS9PM sn5Ul[R "[ U5Vs/sHn[R"U5PM sn5Ul [R"X25Ul gs snfs snf)NTr2silu)act_fn) r5r6r8Linearin_proj ModuleListrangerlayersr!normsout_proj)r<rr{rr_r>s r?r6ChromaApproximator.__init__s yy$? mmW\]eWf gWfRS &zf MWf g  ]]E(O#TOqBJJz$:O#TU  *6  h#Ts C  CcURU5n[URUR5Hup#X"U"U55-nM UR U5$rj)rziprrr)r<rBlayerrs r?rSChromaApproximator.forwardsM LLO TZZ8LEE%(O#A9}}QrA)rrrr)) rUrVrWrXrZr6rSr_r`ras@r?rrs37s7S7c7S77  rArc ^\rSrSrSS\S\S\S\4U4SjjjrSS\RS\RS \ \ \R\R4S \ \RS \ \ \ \ 4S \R4 S jjrSrU=r$)ChromaSingleTransformerBlockrJnum_attention_headsattention_head_dim mlp_ratioc >[TU]5 [X-5Ul[ U5Ul[ R"XR5Ul[ R"SS9Ul [ R"XR-U5Ul [5(aSSK Jn Sn[SSU5 U"5nO [!5n[#UUUUSUS SS 9Ulg) Ntanh) approximater)FluxAttnProcessor2_0_NPUzDefaulting to FluxAttnProcessor2_0_NPU for NPU devices will be removed. Attention processors should be set explicitly using the `set_attn_processor` method. npu_processorz0.34.0Tr-) query_dimdim_headheadsr{r2 processorr0pre_only)r5r6rZmlp_hidden_dimrcr:r8rproj_mlpGELUact_mlpproj_outrattention_processorrrr$r#attn) r<rJrrrrdeprecation_messagerr>s r?r6%ChromaSingleTransformerBlock.__init__s !#/26s;  #':':; ww62  #(;(;";SA ! # # FR  ox1D E02I)+I!'%   rA hidden_statestembimage_rotary_embattention_maskjoint_attention_kwargsrFcUnURXS9upxURURU55n U=(d 0nUbUSS2SSSS24USS2SSS2S4-nUR"SUUUS.UD6n [R "X/SS9nUR S5nXRU5-nXa-nUR[R:XaURSS5nU$) Nr7)rrrrrIr" ) r:rrrr[rrrr^float16clip) r<rrrrrresidualnorm_hidden_statesgatemlp_hidden_states attn_outputs r?rS$ChromaSingleTransformerBlock.forwards!#'99]9#E  LL7I)JK!7!=2  %+AtT1,<=qRVXY[_O_@``Nii ,-) %   ;"BJ ~~a }}];;  0   %-- /)..vu=MrA)rrrr:rr)g@NNN)rUrVrWrXrZfloatr6r[r\rrrstrrrSr_r`ras@r?rrs # # !#  #  # # RIM15;? ||ll#5u||)C#DE  !.  !)c3h 8  rArcH^\rSrSrSS\S\S\S\S\4 U4SjjjrSS\RS \RS \RS \ \ \R\R4S \ \RS \ \ \\ 4S\ \R\R44SjjrSrU=r$)ChromaTransformerBlockirJrrqk_normr0c F>[TU]5 [U5Ul[U5Ul[ UUUUUSS[ 5US9 Ul[R"USSS9Ul [XSS9Ul [R"USSS9Ul [XSS9Ulg)NFT) radded_kv_proj_dimrrr{context_pre_onlyr2rr0r-r.zgelu-approximate)rJdim_out activation_fn)r5r6r&norm1 norm1_contextr#r$rr8r9norm2rff norm2_context ff_context)r<rJrrrr0r>s r?r6ChromaTransformerBlock.__init__s 1#6 9#>!!'%"')   \\#%TJ #BTU\\#%TR%#J\]rArencoder_hidden_statesrrrrrFcUSS2SS24USS2SS24pURXS9uppn URX(S9upnnnU=(d 0nUbUSS2SSSS24USS2SSS2S4-nUR"S U UUUS.UD6n[U5S:XaUunnO[U5S:XaUunnnU R S5W-nUU-nUR U5n U SU SS2S4--U SS2S4-n UR U 5nU R S5U-nUU-n[U5S:XaUW-nUR S5W-nUU-nURU5nUSUSS2S4--USS2S4-nURU5nUUR S5U--nUR[R:XaURSS5nX!4$) NrHr)rrrrrrr"rrr) rrrlenrrrrrr^r[rr)r<rrrrrrtemb_imgtemb_txtrrOrPrQrRnorm_encoder_hidden_states c_gate_msa c_shift_mlp c_scale_mlp c_gate_mlpattention_outputsrcontext_attn_outputip_attn_output ff_outputcontext_ff_outputs r?rSChromaTransformerBlock.forward3s\"!RaR%[$q!"u+(GKzzR_zGnDiHW[WiWi !XjX T" [*"8!=2  %+AtT1,<=qRVXY[_O_@``N!II ,"<-)  %    !Q &/@ ,K, " #q (?P $>vu$M !$33rA)rrrrrrr)rpr-r)rUrVrWrXrZrrr6r[r\rrrrrSr_r`ras@r?rrs" ^ ^!^ ^  ^  ^^FIM15;?<4||<4 %||<4ll <4 #5u||)C#DE <4 !. <4!)c3h 8<4 u||U\\) *<4<4rArc^\rSrSrSrSrSS/rSS/rSS/r\ S"S\ S \ S \ \ S \ S \ S \ S\ S\ S\ \ S4S\ S\ S\ 4U4Sjjj5r S#S\RS\RS\R S\RS\RS\RS\ \\\4S\S\S\\R\44S jjrS!rU=r$)$ChromaTransformer2DModelira The Transformer model introduced in Flux, modified for Chroma. Reference: https://huggingface.co/lodestones/Chroma Args: patch_size (`int`, defaults to `1`): Patch size to turn the input data into small patches. in_channels (`int`, defaults to `64`): The number of channels in the input. out_channels (`int`, *optional*, defaults to `None`): The number of channels in the output. If not specified, it defaults to `in_channels`. num_layers (`int`, defaults to `19`): The number of layers of dual stream DiT blocks to use. num_single_layers (`int`, defaults to `38`): The number of layers of single stream DiT blocks to use. attention_head_dim (`int`, defaults to `128`): The number of dimensions to use for each attention head. num_attention_heads (`int`, defaults to `24`): The number of attention heads to use. joint_attention_dim (`int`, defaults to `4096`): The number of dimensions to use for the joint attention (embedding/channel dimension of `encoder_hidden_states`). axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`): The dimensions to use for the rotary positional embeddings. Trr pos_embedr: patch_size in_channels out_channels num_layersnum_single_layersrrjoint_attention_dimaxes_dims_rope.approximator_num_channelsapproximator_hidden_dimapproximator_layersc l>[TU]5 U=(d UUlXv-Ul[ SU S9Ul[ U S-SU-SU--S-S9Ul[U URU U S9Ul [R"XR5Ul [R"X R5Ul [R"[U5V s/sHn [!URUUS 9PM sn 5Ul[R"[U5V s/sHn [%URUUS 9PM sn 5Ul[)URURS S S 9Ul[R"URX-UR-S S9UlS Ulgs sn fs sn f)Ni')thetaaxes_dimr r)rzr{)rr{rr)rJrrFr-r.Tr)r5r6r inner_dimrrrxtime_text_embedrdistilled_guidance_layerr8rcontext_embedder x_embedderrrrtransformer_blocksrsingle_transformer_blocksrlnorm_outrgradient_checkpointing)r<rrrrrrrrrrrrrr>s r?r6!ChromaTransformer2DModel.__init__s (7K,A%ENKG2a7))EJ,>>B );,NN.( ) %!# *=~~ N))K@"$--z*   +A '(;'9 +   # *,01   2A -(;'9 2   * &; NNDNNu$   $..*2IDL]L]2]dhi &+#5  s F, F1rrrCimg_idstxt_idsrr return_dictcontrolnet_blocks_repeatrFc Ub#UR5nURSS5n OSn [(a [X 5 O+Ub(UR SS5b[ R S5 URU5nURUR5S-nURU5n URU 5nURU5nURS:Xa[ R S5 USnURS:Xa[ R S 5 USn[R"XT4SS 9nUR!U5nUb;S U;a5URS 5nUR#U5nUR%S U05 ['UR(5GH5unnS[+UR,5-nUS [+UR(5--nUS U--nUS U--n[R"USS2UUS -24USS2UUS -244SS 9n[R."5(a)UR0(aUR3UXUUU5up!O U"UUUUUUS9up!UcM[+UR(5[+U5- n[5[6R8"U55nU (aXU[+U5--nGM+XUU--nGM8 [R"X!/SS 9n['UR,5HunnSU-nUSS2UUS-24n[R."5(a&UR0(aUR3UUUU5nO U"UUUUUS9nU cMh[+UR,5[+U 5- n[5[6R8"U55nUSS2UR:SS2S4U UU--USS2UR:SS2S4'M USS2UR:SS2S4nUSS2SS24nUR=UU5nUR?U5n[(a [AX 5 U (dU4$[CUS9$)a$ The [`FluxTransformer2DModel`] forward method. Args: hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): Input `hidden_states`. encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. timestep ( `torch.LongTensor`): Used to indicate denoising step. block_controlnet_hidden_states: (`list` of `torch.Tensor`): A list of tensors that if specified are added to the residuals of transformer blocks. joint_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain tuple. Returns: If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a `tuple` where the first element is the sample tensor. Nrug?z\Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective.rrzrPassing `txt_ids` 3d torch.Tensor is deprecated.Please remove the batch dimension and pass it as a 2d torch TensorrzrPassing `img_ids` 3d torch.Tensor is deprecated.Please remove the batch dimension and pass it as a 2d torch TensorrIip_adapter_image_embedsip_hidden_statesrHr")rrrrrr)rrrrr.)sample)"copypoprrgetloggerwarningrrsr^r rrndimr[rrencoder_hid_projupdate enumeraterrris_grad_enabledr_gradient_checkpointing_funcrZnpceilrrrrr)r<rrrCrrrrcontrolnet_block_samplescontrolnet_single_block_samplesrr lora_scaler pooled_tembidsrrr index_blockblock img_offset txt_offsetimg_modulationtext_modulationrinterval_control start_idxoutputs r?rS ChromaTransformer2DModel.forwardsL " -%;%@%@%B "/33GSAJJ   d /%16L6P6PQXZ^6_6kr 6 ;;}223d:((2 33I> $ 5 56K L <<1  NNU ajG <<1  NNU ajGii*2>>#. ! -2KOe2e&<&@&@AZ&[ ##445LM  " ) )+=?O*P Q"+D,C,C"D KS!?!?@@J#a#d.E.E*F&FFJ'!k/9N(1{?:O99>NQ4F#F FG?_q5H#H HI D$$&&4+F+F7;7X7X=GWYg84%} 8="/*?%5#1+A 84%(3#&t'>'>#?#F^B_#_ #&rww/?'@#A +%sSkOlAl(mm"%2[\lMl4m$mMK#EL #8"HaP "+D,J,J"K KKIq)i!m";;)D)DQ)G)I3&NO 1bc6" mT: }-    19 'v66rA) rrrr rrrrrr rr) r"@N&i)8rAr;ir) NNNNNNNNTF)rUrVrWrXrY _supports_gradient_checkpointing_no_split_modules_repeated_blocks _skip_layerwise_casting_patternsr rZrrr6r[r\r]rrrboolrrrSr_r`ras@r?rrrs6(,$13QR02PQ(3V'<$&*!#"%#%#'*6)+'+#$>,>,>,sm >,  >,  >, >,!>,!>,c3h>,$'>,"%>,!>,>,F/3%) $ $'+;?!%(, ).e7||e7 %||e7"" e7  e7  e7 e7!)c3h 8e7e7#'e7 u||55 6e7e7rAr)=typingrrrrrnumpyr*r[torch.nnr8configuration_utilsr r loadersr r r utilsrrrrrutils.import_utilsrutils.torch_utilsr attentionrr cache_utilsr embeddingsrrrrmodeling_outputsrmodeling_utilsr normalizationrr r!transformer_fluxr#r$ get_loggerrUr"Moduler&rcrlrxrrrrrrAr?rXs 54 B^^aa853$cc7'SS>   H %%;299%;P:))X,ryy,@  &D299DDN[4RYY[4[4|O7!O7rA