
    oi3%                        S SK 7  SSKrS SKJr  S SK JrJr  S SKJrJr  SSK	J
r
JrJrJrJrJr  SSKJrJr  \R(                  R*                  R,                  rSS	 jr         SS
\R2                  S\\   S\\R2                     S\\R8                     S\\\R2                        S\\   S\\   S\\   S\\R8                     S\\\R2                  \R2                  4      4S jjr " S S\5      r g)   )*    N)__version__)LlamaRotaryEmbedding!LlamaLinearScalingRotaryEmbedding)Qwen3Attention_fast_forwardFastQwen3Model)Qwen3MoeAttentionQwen3MoeSparseMoeBlockQwen3MoeMLPQwen3MoeDecoderLayerQwen3MoeModelQwen3MoeForCausalLM)Version
_get_dtypec                 v   UR                   u  pEnUR                  SU5      n[        U R                  XS9n[	        US[
        R                  S9n[
        R                  XR                  SS9u  pXR                  SSS9-  nUR                  UR                  5      n[
        R                  XE-  U4[
        R                  UR                  S9n
[
        R                  R                  R!                  XR"                  S9R%                  S	S
S5      n['        U R"                  5       H~  nU R(                  U   n[
        R+                  X   5      u  pUS U4   R-                  SU5      nU" U5      XUS 4   -  nU
R/                  SUUR                  UR                  5      5        M     U
R-                  XEU5      n
X4$ )N)out)dimdtype)r   T)r   keepdim)r   device)num_classes   r   r   )shapeviewfast_linear_forward	gate_projtorch_nn_functional_softmaxtorchfloat32topktop_ksumtor   zerosr   nn
functionalone_hotnum_expertspermuterangeexpertswherereshape
index_add_)selfX	temp_gatetemp_upbszseq_lenhdrouter_logitsrouting_weightsselected_expertsfinal_Xexpert_mask
expert_idxexpert_layeridxtop_xcurrent_state	current_Xs                     R/home/james-whalen/.local/lib/python3.13/site-packages/unsloth/models/qwen3_moe.py#Qwen3MoeSparseMoeBlock_fast_forwardrD   4   s    wwC"	r2A'M 2RO ).

?JJVX
(Y%O**t*DDO%((1Okk3="-u}}qxxkXG ((%%--(8(8 . gaA 
 D,,-
||J/[[!89

 $+..r26'/d:J*KK 	 	1eY\\!''%:; . oocB/G!!    hidden_statescausal_maskattention_maskposition_idspast_key_valueoutput_attentionsoutput_router_logits	use_cachepadding_maskposition_embeddingsc                 H   UnU(       a  [        U S5      (       aw  Un[        U R                  U5      nU R                  UUUUUUUU	U
U R                  S9
u  pnX-  nUn[        U R
                  U5      n[        U R                  U5      u  nnX-  nOfUn[        U R                  U5      nU R                  UUUUUUUU	U
S9	u  pnX-   nUn[        U R
                  U5      nU R                  U5      u  nnX-   nU4nU(       a  UU4-  nU(       a  UU4-  nU(       a  UU4-  nU$ )N_flag_for_generation)
rF   rG   rH   rI   rJ   rK   rM   rN   rO   rQ   )	rF   rG   rH   rI   rJ   rK   rM   rN   rO   )	hasattrfast_rms_layernorm_inferenceinput_layernorm	self_attnrQ   post_attention_layernormrD   mlpfast_rms_layernorm)r1   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   argskwargsresidualself_attn_weightspresent_key_valuer8   outputss                     rC   !Qwen3MoeDecoderLayer_fast_forwardr_   a   s    HW$  !4  -
 ?Cnn)%+'+ 1!'"5#'#<#< ?M ?
;*; 	! !4))=
 (KHHm(
$} 	! *4+?+?O>Bnn)%+'+ 1!'"5 ?M 
?
;*; !0 !*4+H+H-X'+xx'>$} 0G%''M##%''NrE   c                   N    \ rS rSr\S 5       r\           SS j5       rSrg)FastQwen3MoeModel   c                     [        S[        [        [        S9u  pU b(  [	        U[        5       5        [        U 5      [        l        [        [        l	        [        [        l	        [        [        l	        [        [        l	        [         ["        l	        [%        [&        5      [(        l	        [*        [,        l	        [/        [(        5        SS Kn[        UR2                  R4                  R6                  l        g )NQwen3Moe)
model_namerope_modulescaled_rope_moduleattention_moduler   )patch_linear_scalingr   r   r
   execglobalseval__init__r   forwardrD   r   fast_swiglu_inferencer   r_   r   LlamaModel_fast_forwardr   CausalLM_fast_forward!LlamaModel_fast_forward_inferencer   PeftModel_fast_forwardPeftModelForCausalLM!fix_prepare_inputs_for_generation0transformers.models.qwen3_moe.modeling_qwen3_moemodels	qwen3_moemodeling_qwen3_moeQwen3MoeRotaryEmbedding)	init_namefunctiontransformerss      rC   	pre_patchFastQwen3MoeModel.pre_patch   s    2#.!B0	
	  79%)-i&$?! *M&! 	 (I$ 7&;-'
# (>$)*=> 	@ ! 	%%88P 	rE   Nc                 N    [         R                  " SU UUUUUUU[        U	U
S.UD6$ )N)re   max_seq_lengthr   load_in_4bittoken
device_maprope_scalingfix_tokenizermodel_patchertokenizer_nametrust_remote_code )FastLlamaModelfrom_pretrainedra   )re   r   r   r   r   r   r   r   r   r   r   rZ   s               rC   r   !FastQwen3MoeModel.from_pretrained   sG     -- 
#+'#')-+ 1
 
 	
rE   r   )zQwen/Qwen3-7Bi   NTN
sequentialNTNNF)__name__
__module____qualname____firstlineno__staticmethodr~   r   __static_attributes__r   rE   rC   ra   ra      sJ    # #J $!!
 
rE   ra   )NN)	NNNNFFFNN)!llamaos_utilsr   r   r   qwen3r   r	   rv   r
   r   r   r   r   r   unsloth_zoo.utilsr   r   r    r'   r(   softmaxr   rD   TensorOptionalBlockDiagonalCausalMask
LongTensorTupleboolr_   ra   r   rE   rC   <module>r      s?    	  ( 2 $hh1199 *"` 6:-1/348(-+0 %/3GKL<<L 12L U\\*	L
 5++,L U5<<01L  ~L #4.L ~L 5++,L "%ell(B"CDL^C
 C
rE   