+hYH SSKJrJrJrJr SSKrSSKJr SSKJ r J r SSK J r SSK Jr SSKJrJrJrJr SSKJr S S KJr S S KJrJr S S KJr S S KJrJr S SK J!r! S SK"J#r# S SK$J%r%J&r& \RN"\(5r)"SS\RT5r+"SS\RT5r,"SS\RT5r-\"SS\RT55r."SS\RT5r/\"SS\#\ \ \\55r0g))AnyDictOptionalTupleN) ConfigMixinregister_to_config)PeftAdapterMixin)FromOriginalModelMixin)USE_PEFT_BACKENDloggingscale_lora_layersunscale_lora_layers)maybe_allow_in_graph) FeedForward)MochiAttentionMochiAttnProcessor2_0) CacheMixin)%MochiCombinedTimestepCaptionEmbedding PatchEmbed)Transformer2DModelOutput) ModelMixin)AdaLayerNormContinuousRMSNormc:^\rSrSrS\4U4SjjrSSjrSrU=r$)MochiModulatedRMSNorm&epscR>[TU]5 Xl[SUS5UlgNrF)super__init__rrnorm)selfr __class__s i/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/models/transformers/transformer_mochi.pyr#MochiModulatedRMSNorm.__init__'s# AsE* cURnUR[R5nUR U5nUbX-nURU5nU$N)dtypetotorchfloat32r$)r% hidden_statesscalehidden_states_dtypes r'forwardMochiModulatedRMSNorm.forward-sU+11%((7  -0  )1M%(()<= r))rr$r+) __name__ __module__ __qualname____firstlineno__floatr#r3__static_attributes__ __classcell__r&s@r'rr&s+E+  r)rc^\rSrSrS S\S\4U4SjjjrS\RS\RS\R4SjrS r U=r $) MochiLayerNormContinuous; embedding_dimconditioning_embedding_dimc>[TU]5 [R"5Ul[R "X!US9Ul[US9Ulg)N)biasr) r"r#nnSiLUsiluLinearlinear_1rr$)r%r@rArrCr&s r'r#!MochiLayerNormContinuous.__init__<s? GGI  "?BB177KL IIa!eooa033EMMBB DttK  r))rIr$rG)h㈵>T) r5r6r7r8intr#r.Tensorr3r:r;r<s@r'r>r>;sY   3 3%( 3 3 ! << !!&  !  ! !r)r>c ^\rSrSrSrSS\S\S\S\SS4 U4S jjjrS \ RS \ RS\ \ R\ R\ R\ R44S jr S r U=r$)MochiRMSNormZeroXzm Adaptive RMS Norm used in Mochi. Parameters: embedding_dim (`int`): The size of each embedding vector. r@ hidden_dimrelementwise_affinerMNc>[TU]5 [R"5Ul[R "X5Ul[SUS5Ulgr!) r"r#rErFrGrHlinearrr$)r%r@rYrrZr&s r'r#MochiRMSNormZero.__init__`s= GGI ii : AsE* r)r0embcZURnURURU55nURSSS9upEpgUR UR [ R55SUSS2S4R [ R5--nUR U5nXXg4$)NrOdim)r,r\rGchunkr$r-r.r/)r%r0r^r2 scale_msagate_msa scale_mlpgate_mlps r'r3MochiRMSNormZero.forwardis,11kk$))C.)3699QA93F0 Y -"2"25=="ABa)TUW[T[J\J_J_`e`m`mJnFno %(()<=  ;;r))r\r$rG)rSF)r5r6r7r8__doc__rTr9boolr#r.rUrr3r:r;r<s@r'rWrWXsbg+ +.1+8=+Z^+ ++ <"\\ <05  < u||U\\5<<E F < [T U]5 XplSU-S-S-UlSU-S-S-Ul[ USU-USS9UlU(d[ USU-USS9UlO[UUUS9Ul[UUUSUSUUU[5SS9 Ul [US 9Ul UR(d [US 9OSUl[U5UlUR(d [US 9OSUl[#XRUSS 9UlSUlU(d[#UURUSS 9Ul[US 9Ul[US 9Ulg) Nr`rrF)rrZ)r@rArrS) query_dimheadsdim_headrCadded_kv_proj_dimadded_proj_biasout_dimout_context_dimrs processorrrD) inner_dimrrrC)r"r#rs ff_inner_dimff_context_inner_dimrWnorm1 norm1_contextr>rrattn1rnorm2 norm2_contextnorm3 norm3_contextrff ff_contextnorm4 norm4_context) r%rbrnrorprqrrrsrr&s r'r#MochiTransformerBlock.__init__sf  0Wq[Q.%&)>%>%Bq$H!%c1s7PUV !1#q;P7PVYns!tD !93+."D  $%'3!1-+-   +s3 CGCXCX2s;^b*3/ CGCXCX2s;^bc->->mbgh)%33+ DO+s3 2s;r)r0encoder_hidden_statestembencoder_attention_maskimage_rotary_embcURX5upgpUR(dURX#5uppOURX#5n URUU UUS9upXR U[ R "U5RS55-nURUSURS5R[ R5-5nURU5nXRU[ R "U 5RS55-nUR(dX RU[ R "W 5RS55-nURUSW RS5R[ R5-5n URU 5nX R!U[ R "W 5RS55-nX4$)N)r0rrattention_maskrO)rrsrrrr.tanhrPrr-r/rrrrrr)r%r0rrrrnorm_hidden_statesrerfrgnorm_encoder_hidden_states enc_gate_msa enc_scale_mlp enc_gate_mlpattn_hidden_statescontext_attn_hidden_states ff_outputcontext_ff_outputs r'r3MochiTransformerBlock.forwards=AJJ}<[9i$$TXTfTf%U Q &m\*.););R>\>\]^>_(`` $$$9r+)r5r6r7r8rirTstrrjr9r#r.rUrrr3r:r;r<s@r'rlrlvs4"%!&<< <<!<< << # <<  <<<<<<<< <<<[TU]5 X-Ulgr+)r"r# target_area)r%rrr&s r'r#MochiRoPE.__init__s &3r)cP[R"XUS-XES9nUSSUSS-S- $)NrOdevicer,r)r.linspace)r%startstopnumrr,edgess r'_centersMochiRoPE._centerss4uC!GFPcr U12Y&!++r) num_framesheightwidthrr,cXURX#-- S-n[R"XUS9nURU*U-S- X&-S- X$U5nURU*U-S- X6-S- X4U5n [R"XxU SS9upn [R "XU /SS9R SS5n U $) Ng?rrij)indexingrrar)rr.arangermeshgridstackview)r%rrrrr,r1thwgrid_tgrid_hgrid_w positionss r'_get_positionsMochiRoPE._get_positions s!!V^4< LL% @ MM6'E/A-v~/A6SX Y MM5&5.1,ema.?PU V!&a$!GKK 8bAFFr1M r)freqsposc[R"URR[R5 [R "SUR [R5UR [R55nSSS5 [R"U5n[R"U5nX44$!,(df  N==f)Nz nd,dhf->nhf) r.autocastrtyper/einsumr-cossin)r%rr freqs_cos freqs_sins r' _create_ropeMochiRoPE._create_ropes ^^ELL--u}} =LLu}}0EuxxPUP]P]G^_E>IIe$ IIe$ ## > =s AC Cpos_frequenciescTURX#XEU5nURX5upX4$r+)rr) r%rrrrrr,rrope_cosrope_sins r'r3MochiRoPE.forward&s4!!*eUK!..D!!r))r)r)NN)r5r6r7r8rirTr#r.rUrrrr,rrrr3r:r;r<s@r'rrs84C43444 ,5<<,*.'+   &   $  &$%,,$U\\$ell$*.'+ " " " "  " & " $ " u||U\\) * " "r)rcD^\rSrSrSrSrS/rSS/r\SS\ S \ S \ S \ S \ S \ S\ \ S\ S\ S\ S\ S\ SS4U4Sjjj5r SS\ RS\ RS\ RS\ RS\ \\ \4S\S\ R4SjjrSrU=r$) MochiTransformer3DModeli4a A Transformer model for video-like data introduced in [Mochi](https://huggingface.co/genmo/mochi-1-preview). Args: patch_size (`int`, defaults to `2`): The size of the patches to use in the patch embedding layer. num_attention_heads (`int`, defaults to `24`): The number of heads to use for multi-head attention. attention_head_dim (`int`, defaults to `128`): The number of channels in each head. num_layers (`int`, defaults to `48`): The number of layers of Transformer blocks to use. in_channels (`int`, defaults to `12`): The number of channels in the input. out_channels (`int`, *optional*, defaults to `None`): The number of channels in the output. qk_norm (`str`, defaults to `"rms_norm"`): The normalization layer to use. text_embed_dim (`int`, defaults to `4096`): Input dimension of text embeddings from the text encoder. time_embed_dim (`int`, defaults to `256`): Output dimension of timestep embeddings. activation_fn (`str`, defaults to `"swiglu"`): Activation function to use in feed-forward. max_sequence_length (`int`, defaults to `256`): The maximum sequence length of text embeddings supported. Trl patch_embedr$N patch_sizernro num_layersrp in_channels out_channelsrqtext_embed_dimtime_embed_dimrrmax_sequence_lengthrMc >[TU]5 X#-n U=(d Un[UUU SS9Ul[ U UU U SS9Ul[ R"[R"SX#S-4S55Ul [5Ul [ R"[U5Vs/sHn[U UUUUU XS- :HS9PM sn5Ul[#U U S S S S 9Ul[ R&"XU-U-5UlS Ulgs snf) N)rr embed_dimpos_embed_type)r@rprrrnrrgrO)rbrnrorprqrrrsFr layer_norm)rZr norm_type)r"r#rrr time_embedrE Parameterr.fullrrrope ModuleListrangerltransformer_blocksrnorm_outrHproj_outgradient_checkpointing)r%rrnrorrprrrqrrrrrr}ir&s r'r# MochiTransformer3DModel.__init__Vs+ '< #2{ %!#  @#"7)) !   "||EJJ;NfgPg7hjm,noK "$--z* +A&!(;'9*?#"/%&q.%8+  # /  $"    )*-D|-ST &+#/ s#Dr0rtimesteprattention_kwargs return_dictc Ub#UR5nURSS5nOSn[(a [X5 O+Ub(UR SS5b[ R S5 URuppn URRn X-nX-nURUUUURS9unnURSSSSS 5RSS5nURU5nURSUS 45RSS5nUR!UR"U UUUR$[&R(S 9n[+UR,5HWunn[&R."5(a+UR0(aUR3UUUUUU5upMKU"UUUUUS 9upMY UR5UU5nUR7U5nUR9XXXS 5nURSS SSS SS5nUR9US XU 5n[(a [;X5 U(dU4$[=US9$)Nr1g?zVPassing `scale` via `attention_kwargs` when not using the PEFT backend is ineffective.) hidden_dtyperrrOrr`rr)r0rrrr)sample)copypopr rgetloggerwarningshapeconfigrrr,permuteflattenr unflattenrrrr.r/ enumerateris_grad_enabledr_gradient_checkpointing_funcrrreshaperr)r%r0rrrrr lora_scale batch_size num_channelsrrrppost_patch_heightpost_patch_widthrrrblockoutputs r'r3MochiTransformer3DModel.forwardsa  '/446 )--gs;JJ   d /+0@0D0DWd0S0_l?L>Q>Q; *e KK " ""K :&*oo  ! "&,, '6' ##&--aAq!<DDQJ ((7 %//J3CDLLQPQR 99     ''-- % "$"9"9:HAu$$&&4+F+F7;7X7X!)*$ 84 48="/*?+A%5 84 4;$ mT:  m4 %--jFWklqst %--aAq!QB &&z2z5Q    19 'v66r))rrrrrrrr) r0i Nrirr)NT)r5r6r7r8ri _supports_gradient_checkpointing_no_split_modules _skip_layerwise_casting_patternsr rTrrr#r.rU LongTensorrrrjr3r:r;r<s@r'rr4sc8(,$01(5v'>$#%"%%)&*!"!%#&>,>,!>, >,  >, # >,>,sm>,>,>,>,>,!>, >,>,L6: Q7||Q7 %||Q7"" Q7 !& Q7 #4S>2 Q7Q7 Q7Q7r)r)1typingrrrrr.torch.nnrEconfiguration_utilsrr loadersr loaders.single_file_modelr utilsr r rrutils.torch_utilsr attentionrattention_processorrr cache_utilsr embeddingsrrmodeling_outputsrmodeling_utilsr normalizationrr get_loggerr5rModulerr>rWrlrrr)r'r.s .- B'?VV5#G$J7';   H %BII*!ryy!: