oi % S SK r S SKrS SKJr S SKJr S SKrS SKrS SKJ r S SK
J r J r S SK
Jr S SKJrJrJr S SKJrJr \R, S \R. S
\R0 S 0rS rS
rS/r \R, /r Sr \S;S j5 r S r S r!S\RD 4S jr#\RH " 5 SS8' S9 H$ r?\2" \8R \=\8R^ S"SS"\?S"S:9 M& gg)> N)contextmanager)partial)
AutoConfig) Llama4ConfigLlama4TextConfig)
Llama4TextMoe)KernelConfigBackward_dWKernelConfigBackward_dXKernelConfigForward)Llama4GroupedGemmTextMoeLlama4TritonTextMoe){Gz?r )MbP?r )h㈵>r z meta-llama/Llama-4-Scout-17B-16E* i 2 c # t # [ X#- 5 [ U 5 S v [ U5 [ X#- 5 g 7fN)print)preludeepiloguechar num_charss c/home/james-whalen/.local/lib/python3.13/site-packages/unsloth/kernels/moe/tests/test_llama4_moe.pyannotated_contextr ' s, $
'N (O $
s 68c F [ R " U 5 nUR $ r )r from_pretrained text_config)model_idconfigs r get_text_configr! 0 s %55h?F c U ( d [ 5 n[ 5 n[ 5 nO\SSKJnJn SSKJn UR S [ Ul UR S [ Ul UR S [ Ul S nS nS nXU4$ )Nr )!_autotuned_grouped_gemm_dW_kernel!_autotuned_grouped_gemm_dX_kernel)&_autotuned_grouped_gemm_forward_kernel)
r r r
grouped_gemm.kernels.backwardr$ r% grouped_gemm.kernels.forwardr& configsNUM_AUTOTUNE_CONFIGS)autotunekernel_config_fwdkernel_config_bwd_dWkernel_config_bwd_dXr$ r% r& s r prep_triton_kernel_traitsr/ 5 s / 1 6 8 6 8
X
3 : :;P> ! & & (D "9C
R[)*+ >>" ;; ;r" y grad_outputmodulec U R U5 UR 5 H u p4UR b M U S35 e g )Nz missing grad!)backwardnamed_parametersgrad)rI rJ rK nameparams r
run_backwardsrR k s? JJ{ ..0 zz%>$~'>>% 1r" m1m2c
U R 5 H= u px[ UR UR U5 R UUUUU SU S3S9 M? g )N:z.grad)r< r= rE rF rG )rN rH rO
get_parameter) rS rT r< r= rE rF rG rP rQ s r _check_gradsrX q sX **, JJ T " ' '! E4&&
-r" c J [ R " [ 5 R $ r )r r LLAMA4_SCOUT_IDr r" r model_configr\ s % %o 6 B BBr" overlap_router_sharedTc U ( a S$ S$ )Nr]
no_overlapr[ xs r rb s q+BlBr" )ids permute_yc U ( a S$ S$ )Nrd no_permute_yr[ r` s r rb rb s q 0Tn0Tr" permute_xc U ( a S$ S$ )Nrg no_permute_xr[ r` s r rb rb s +*N*Nr" r+ c U ( a S$ S$ )Nr+ manualr[ r` s r rb rb s a
(EX(Er" seqlenc SU 3$ )Nzseqlen=r[ r` s r rb rb s
wqc]r" dtyper\ bsc [ R " [ 5 SnUR n [ U u p[ [ XXS9n[ [ XXS9n[ U5 R XS9n[ XeS9R XS9nUR U5 UR U5 [ R " XqXUSS9nUR 5 R 5 R! 5 nUR 5 R 5 R! 5 nU" U5 u nnU" U5 u nnUR" UR" :X d UR" SUR" 35 e[% S5 U" UUS S
9 U" [' U5 US S
9 S S S 5 [) U5 u nnn[+ UUUUUUUUS 9R XS
9n U R U5 U R U5 U " U5 u n n [% S5 U" UU SS
9 U" [' U5 U SS
9 S S S 5 [ R, " U5 n [/ UU U5 [/ UU U5 [% S5 U" UUSS
9 S S S 5 [/ U U U 5 [% S5 U" UU SS
9 S S S 5 g ! , ( d f GN = f! , ( d f N= f! , ( d f N[= f! , ( d f g = f)Ncuda)r< r= rE rF )rn device)r] T)rn rr
requires_gradz != z(Testing torch grouped gemm Llama4TextMoe
y_torch_gg)rG routing_torch_gg)r] rg rd r+ r, r- r. )rr rn z1Testing triton grouped gemm Llama4TextMoe forwardy_tritonrouting_tritonz/Testing torch group gemm Llama4TextMoe backwardtorch_ggz0Testing triton group gemm Llama4TextMoe backwardtriton)rC manual_seedSEED hidden_size
TOLERANCESr rH rX r tor copy_weights
check_weightsrandndetachclonerequires_grad_shaper r6 r/ r
randn_likerR ) rn rl r+ rg rd r] r\ ro rr rE rF
hidden_dimr< r=
check_diff check_grads
llama4_ref
llama4_gg_refx_ref
x_torch_ggx_tritony_ref routing_refrt ru r, r- r.
llama4_tritonrv rw ref_grads r test_llama4_refr s :
F ))JE"JDD9J TIK
|,///OJ -bb( z*
+ KK
JPT
E %%'668J||~##%446H#E*E;#0#< J ;;*** *Ru{{m4
@P@P?Q,RR * E F5*L9 K (*:BT
G "(+ B+-A ( 5 - 3 3 b&b( z*
+ ,X 6 Hn N O5(*5?;/ GWX
P &H%:.*h
6 L MJ
Z@
N (Hm4 M NJ
X>
O NG
G F0
P O
N M
O Ns0 ,J
2J J-8 J>
J
J*-
J;>
K __main__z--seqlen)typedefaultz--dtypebfloat16float16)r choicesr r )FT)rl r\ rn r+ rg rd r] rF )zPassed!-P ).6fFr: ) rq r F)Aargparsesys
contextlibr functoolsr pytestrC transformersr transformers.models.llama4r r *transformers.models.llama4.modeling_llama4r grouped_gemm.kernels.tuningr r
r (grouped_gemm.reference.layers.llama4_moer r
r r floatr} rZ r{ SEQ_LENSDTYPESr* r r! r/ Tensorr6 no_gradrH nnModulerR rX fixturer\ mark parametrizestrrn boolintr __name__ArgumentParserparser add_argument
parse_argsargsgetattrvars args_dictr r __annotations__overlaprl r[ r" r r sn
%
# E D
NNL MM< KK
5 6
..
I<
u||
<
<
< <$?U\\ ? ?ehhoo ?
* C C
DM
B
% &T % N E 83JK&4
Q? ;;Q? Q? Q?
Q?
Q? #Q? Q? 5 L Q?h z
$
$
&F
3$?
#*i)@J D
+DJT
I H$3H$=K!= [[&JJ $+
! r"