
    h                        S SK Jr  S SKrS SKJr  S SKJr  S SKJ	r	J
r
  S SKJr    SS\R                  S\R                  R                  S\R                  S\R                  R                  S	\R                   S
\	S\S\4S jjrS\R                  S\R                  S\R                  R                  S\R                  S\R                  R                  S	\R                   4S jrg)    )ListN)Float8LinearConfigScalingType)(precompute_float8_dynamic_scale_for_fsdp	ref_model	ref_optim
fsdp_model
fsdp_optim	local_inpconfig
precomputecompile_transformer_blockc	                    [        UR                  5       UR                  5       5       HD  u  pU R                  U	R                  U
R                  SU	R                   SU
R                   3S9  MF     [	        S5       GHZ  n/ nX4X444 GH#  u  pUR                  US-  S:H  S9  UR                  U" U5      R                  5       5        US   R                  5         XL ax  UR                  5        Hd  nUR                  (       d  M  [        R                  " UR                  5        UR                  R                  [        R                  " 5       5        Mf     UR                  5         XL d  M  U(       d  M  UR                  R                   ["        R$                  L d  GM  ['        U5        GM&     U R                  US   US	   S
U SUS    SUS	    3S9  GM]     g )Nzref_param.requires_grad: z, fsdp_param.requires_grad: msg
      r   set_to_none   iter: , loss-ref: , loss-fp8: )zip
parametersassertEqualrequires_gradrange	zero_gradappendsumbackwarddist
all_reducegraddiv_get_world_sizestepcast_config_weightscaling_typer   DYNAMICr   )test_clsr   r   r	   r
   r   r   r   r   	ref_param
fsdp_paramiter_idxlossesmodeloptimparams                   ^/home/james-whalen/.local/lib/python3.13/site-packages/torchao/testing/training/fsdp2_utils.pycheck_parity_no_mpr6      s    "%Y%9%9%;Z=R=R=T!U	##$$+I,C,C+DD`akayay`z{ 	 	
 "V "I%''3j5MNLEOOA):O<MM%	*..012J!!"--/E***

3

(;(;(=> 0
 JJL#J--::k>Q>QQ8?! O$ 	1I1I
,vaykfQi[Q 	 	
)     ref_model_bf16c                 J   [        S5       GH  n/ nX#4XE44 GH\  u  pU
R                  US-  S:H  S9  UR                  U	" U5      R                  5       5        US   R	                  5         XL a  [        UR                  5       UR                  5       5       Hy  u  p[        R                  " UR                  5        UR                  R                  [        R                  " 5       5        UR                  R                  5       Ul	        S Ul	        M{     U
R                  5         [        UR                  5       UR                  5       5       H$  u  pUR                  5       R                  U5        M&     GM_     U R!                  US   US   SU SUS    S	US    3S
9  GM     g )Nr   r   r   r   r   r   r   r   r   r   )r   r    r!   r"   r#   r   r   r$   r%   r&   r'   r(   floatr)   detachcopy_r   )r-   r   r8   r   r	   r
   r   r0   r1   r2   r3   
param_bf16
param_fp32s                r5   check_parity_bf16_mpr?   B   sy    "I%''$
LE OOA):O<MM%	*..012J!&.1"--/1E1E1G/*J OOJOO4OO(()<)<)>?&0oo&;&;&=JO&*JO/ JJL*-$$&(A(A(C+&
 !!#))*5+!
( 	1I1I
,vaykfQi[Q 	 	
- r7   )FF)typingr   torchtorch.distributeddistributedr$   torch.nnnntorchao.float8.configr   r   torchao.float8.fsdp_utilsr   Moduler3   	OptimizerTensorboolr6   r?    r7   r5   <module>rM      s         O &+,
yy,
 {{$$,
 			,

 %%,
 ||,
 ,
 ,
  $,
^#
yy#
 II#
 {{$$	#

 		#
 %%#
 ||#
r7   