
    oi                         S SK r / SQ/ SQ/ SQ/ SQ\ R                  \ R                  \ R                  \ R                  // SQ/ SQ/ SQ/ S	Q/ S
Q/ SQ/ SQ/ SQ/ SQ4S jrg)    N)      r   r   ){   C  r   r   )i  i  im  iW  )i @     r   i   )cudar   r   r   )      ?g      0@       @g      @)        r	   r   r   )r   r   r
   r   )g      $@r   r   g      I@)iO  iP  iQ  iR  )r   g?g?g333333?)FFTT)FTFTc                     [        X#XEXgUXUXX5       GHg  u  nnnnnnnnnnnnnn0 nUUS'   UUS'   UUS'   [        R                  R                  U5        [        R                  " UU4UUS9n[        R                  " UUUS9n [        R
                  " SUUU4US9n![        R                  " UUU4UUS9n"U"R                  S5        U(       a  UR                  S5        U(       a  U R                  S5        [        R                  " U!R                  5       [        R                  US9n#SU#S [        U#R                  5       U-  5      & [        R                  " U#R                  5       5      n$U#U$   n#S	U!R                  5       U#'   U!S	:g  R                  5       n%U " SU"UU U!U%US
.UD6n&U&S   R                  5         U&S   S   n&U"R                  R!                  5       R#                  5       n'U"?U(       a*  UR                  R!                  5       R#                  5       n(U?U(       a*  U R                  R!                  5       R#                  5       n)U ?U"  SU"UU U!U%USSS.UD6n*U*R                  5         [        R$                  R'                  U&U*SS9  U"R                  R!                  5       R#                  5       n+U"?[        R$                  R'                  U+U'SS9  U(       aI  UR                  R!                  5       R#                  5       n,U?[        R$                  R'                  U,W(SS9  U(       d  GM  U R                  R!                  5       R#                  5       n-U ?[        R$                  R'                  U-W)SS9  GMj     g )Nlogit_scale_multiplylogit_scale_dividelogit_softcapping)dtypedevicer   )lowhighsizer   Tr   i)hidden_stateslm_head_weightlm_head_biaslabelsn_itemsscalinggMbP?)r   r   r   r   r   r   	target_gbtorch_compilezLoss diff too high)msgz Hidden States grad diff too highzLM Head grad diff too highzLM Bias grad diff too high )N)ziptorchr   manual_seedrandnrandintrequires_grad_zerosnumelboolintrandpermravelsumbackwardgraddetachclonetestingassert_close).compute_fused_ce_lossunsloth_fused_ce_lossbszsqlenshdsvocab_sizesdtypesdevicesscalingslogit_scale_multiplyslogit_scale_divideslogit_softcappingsseedsmask_ratioslm_head_requires_gradslm_bias_requires_gradsbszqlenhd
vocab_sizer   r   r   r   r   r   seed
mask_ratiolm_head_requires_gradlm_bias_requires_gradkwargsr   r   	input_idsr   maskpermutationr   old_lossold_hidden_states_gradold_lm_head_gradold_lm_bias_gradnew_lossnew_hidden_states_gradnew_lm_head_gradnew_lm_bias_grads.                                                 X/home/james-whalen/.local/lib/python3.13/site-packages/unsloth_zoo/fused_losses/tests.pytest_fused_ce_lossrW      sv   . 
Sv4F	
	T2z5&'02Cj4 )=%&'9#$&7"#

t$j"%5uvV{{:uvNMM*c4[[ab	S$OUVT$$T* ."?"?"E ,"="=d"C {{9??,ejj6R./*c$**,z)*+nnTZZ\2K "&	$$))+( 
)+'
 
 	A;q>!.!3!3!:!:!<!B!B!D -2299;AAC# +00779??A!(
)+' 
 
 	""8X=Q"R!.!3!3!:!:!<!B!B!D""#9;QY{"| -2299;AAC#MM&&'79IQm&n  +00779??A!MM&&'79IQm&nO
P 	    )r    float32rW   r   rX   rV   <module>rZ      s[   " 
 !
!?mmU]]EMM5==J/%1/0%'88!_	@ rX   