
    hk`                        S SK r S SKrS SKrS SKrS SKJrJrJrJr  S SK	J
r
  S SKJrJr  S SKrS SKJr  S SKJrJr  S SKJrJr  S SKJr  S S	KJr  S S
KJr  S SKJrJr   S r SS jr!S r"S r#SS jr$ " S S\
RJ                  5      r& " S S\
RJ                  5      r' " S S\5      r( " S S\
RJ                  5      r)\
RT                  " \&5        \
RT                  " \'5        \
RT                  " \(5        \+S:X  a  \RX                  " 5         gg)    N)
DeviceMeshDTensor	ReplicateShard)common_utils)DTensorTestBase
with_comms)AOBaseConfig)AffineQuantizedTensorto_affine_quantized_intx)Int8WeightOnlyConfig	quantize_)MappingType)_QUANTIZE_CONFIG_HANDLER)LlamaModelsLlama4Experts)DummyModuleget_compute_capabilityc                     ^ ^ SS K mU U4S jnU$ )Nr   c                    >^  UU U4S jnU$ )Nc                  Z   > [        5       T:  a  TR                  " ST 35      eT" U 0 UD6$ )Nz Compute capability is less than )r   SkipTest)argskwargsmin_capability	test_funcunittests     O/home/james-whalen/.local/lib/python3.13/site-packages/torchao/testing/utils.pywrapperHskip_if_compute_capability_less_than.<locals>.decorator.<locals>.wrapper=   s>    %'.8''6~6FG  d-f--     )r   r   r   r   s   ` r   	decorator7skip_if_compute_capability_less_than.<locals>.decorator<   s    	. r    r   )r   r"   r   s   ` @r   $skip_if_compute_capability_less_thanr%   9   s     r    c                 X   ^ ^ SSK mU U4S jn[        T 5      (       a  T nSm U" U5      $ U$ )zDecorator to skip tests on ROCm platform with custom message.

Args:
    message (str, optional): Additional information about why the test is skipped.
r   Nc                 L   >^  [         R                  " T 5      U UU4S j5       nU$ )Nc                     > [         R                  R                  b"  SnT(       a  UST 3-  nTR                  U5        T" U 0 UD6$ )NzSkipping the test in ROCmz: )torchversionhipskip)r   r   skip_messagefuncmessagepytests      r   r   0skip_if_rocm.<locals>.decorator.<locals>.wrapperR   sG    }}  ,: b	N2LL)(((r    	functoolswraps)r.   r   r/   r0   s   ` r   r"   skip_if_rocm.<locals>.decoratorQ   s%    			) 
	) r    )r0   callable)r/   r"   r.   r0   s   `  @r   skip_if_rocmr7   I   s4     
 r    c                     ^ SS K mU4S jn U $ )Nr   c                 J   >^  [         R                  " T 5      U U4S j5       nU$ )Nc                  ~   > [         R                  R                  5       (       d  TR                  " S5      eT" U 0 UD6$ )NzNo cuda available)r)   cudais_availabler   )r   r   r   r   s     r   r   3skip_if_no_cuda.<locals>.decorator.<locals>.wrapperi   s8    ::**,,''(;<<d-f--r    r2   r   r   r   s   ` r   r"   "skip_if_no_cuda.<locals>.decoratorh   s%    		#	. 
$	.
 r    r$   r"   r   s    @r   skip_if_no_cudarA   e   s     r    c                     ^ SS K mU4S jn U $ )Nr   c                 J   >^  [         R                  " T 5      U U4S j5       nU$ )Nc                  L   >  SS K nT" U 0 UD6$ !   TR                  " S5      e= f)Nr   zNo gemlite available)gemliter   )r   r   rE   r   r   s      r   r   6skip_if_no_gemlite.<locals>.decorator.<locals>.wrapperx   s8    @ d-f--@''(>??s    #r2   r>   s   ` r   r"   %skip_if_no_gemlite.<locals>.decoratorw   s%    		#	. 
$	. r    r$   r@   s    @r   skip_if_no_gemliterH   t   s    	 r    c                 P   U R                   R                  5        GH  u  pVUR                  S5      (       d  M  [        R                  " U5      U4S j5       n[
        R                  " UR                   5      Ul         Ub&  [        Xd5      (       a  [        R                  " U5      nU=(       a    UR                  U5      nUbN  X(R                  ;   a?  UR                  (       a  [        R                  " S5      O[        R                  n	U	" U5      n[        X SU 3U5        GM
     g )Ntest_c                     U" U 5      $ Nr!   )selfvalues     r   new_testcopy_tests.<locals>.new_test   s    T{"r    zSkipped!_)__dict__items
startswithr3   r4   copydeepcopyhasattrr   expectedFailuregetsuffixesis_skipr,   setattr)
my_cls	other_clssuffixtest_failures
xfail_propnamerN   rO   tf	skip_funcs
             r   
copy_testsre      s    ,,.??7## __U#%* # $# !%enn =H%'%*D*D#33H=:=#4#4T#:B~&KK"7 zz MM*-!11 
 %X.Iq18<7 /r    c                   h   \ rS rSrS/\R
                  R                  5       (       a  S/O/ -   r\R                  \R                  \R                  /r\r\r\R"                  S\R$                  S.rSrS r\R.                  " S\5      \R.                  " S	\5      S
 5       5       r\R.                  " S\5      \R.                  " S\5      S 5       5       r\R.                  " S\5      \R.                  " S	\5      S 5       5       r\R.                  " S\5      \R.                  " S	\5      S 5       5       rSrg)TorchAOBasicTestCase   cpur;          mapping_type
block_sizetarget_dtype(   c                    [         R                  " SS5      nU R                  " U40 U R                  D6nUR	                  5       u  p4U Vs0 s H  oU[        X%5      _M     nnUR                  5       nUR                  5       nU R                  R                  XdXx5      n	U R                  UR                  5       U	R                  5       5        g s  snf )N      )r)   randn
FACTORY_FNr   __tensor_flatten__getattrsizestrideTENSOR_SUBCLASS__tensor_unflatten__assertEqual
dequantize)
rM   	hp_tensor	lp_tensortensor_data_name_dicttensor_attributesrb   tensor_data_dict
outer_sizeouter_stridereconstructeds
             r   test_flatten_unflatten+TorchAOBasicTestCase.test_flatten_unflatten   s    KK3'	OOI==	3<3O3O3Q07L
7Lt')**7L 	 
 ^^%
 ''),,AA
 	--/1I1I1KL
s   Cdevicedtypec                 j    [         R                  " SSXS9nU R                  " U40 U R                  D6  g )Nrs   rt   r   r   )r)   ru   rv   r   )rM   r   r   r   s       r   test_hp_tensor_device_dtype0TorchAOBasicTestCase.test_hp_tensor_device_dtype   s+     KK3vC		1T[[1r    device1device2c                    [         R                  " SSU[         R                  S9nU R                  " U40 U R                  D6nUR                  US9  [         R                  " SSU[         R                  S9nU R                  " U40 U R                  D6nUR                  U5        [         R                  " SSU[         R                  S9nU R                  " U40 U R                  D6nUR                  5         [         R                  " SSU[         R                  S9nU R                  " U40 U R                  D6nUR                  5         g)zuNote: this should be parametrized with device1 and device2
e.g. device1 = ["cpu", "cuda"], device2 = ["cpu", "cuda"]
rs   rt   r   )r   N)r)   ru   bfloat16rv   r   tor;   ri   )rM   r   r   r   r   s        r   test_device1_to_device2,TorchAOBasicTestCase.test_device1_to_device2   s     KK3wennM	OOI==	G$KK3wennM	OOI==	WKK3wennM	OOI==	KK3wennM	OOI==	r    c                     [         R                  " SSXS9nU R                  " U40 U R                  D6nUR	                  5       nU R                  UR                  S5        g )Nrs   rt   r   )rt   rs   )r)   ru   rv   r   tr}   shape)rM   r   r   r   r   s        r   test_transpose#TorchAOBasicTestCase.test_transpose   sL     KK3vC	OOI==	KKM	(3r    c                    [         R                  " SSXS9nU R                  " U40 U R                  D6n[         R                  " SSXS9n[         R                  R
                  R                  XS5      n[         R                  R
                  R                  XT5      nU R                  [        R                  R                  R                  Xg5      U R                  5        g )Nrs   rt   r   rl   )r)   ru   rv   r   nn
functionallinearassertGreatertorchaoquantizationutilscompute_errorLINEAR_MIN_SQNR)rM   r   r   r   r   hp_act_tensorhp_reslp_ress           r   test_linear TorchAOBasicTestCase.test_linear   s     KK3vC	OOI==	BFH$$++ME$$++ME  &&44VD  	
r    r!   N)__name__
__module____qualname____firstlineno__r)   r;   r<   COMMON_DEVICESfloat32float16r   COMMON_DTYPESr   r{   r   rv   r   
ASYMMETRICuint8r   r   r   r   parametrizer   r   r   r   __static_attributes__r!   r    r   rg   rg      sE   WEJJ,C,C,E,E2NN]]EMM5>>BM+O)J#..F OM h7g}52 6 82 i8i8 9 9( h7g}54 6 84 h7g}5

 6 8

r    rg   c                   f   \ rS rSrS/\R
                  R                  5       (       a  S/O/ -   r\R                  \R                  \R                  /r\r\r\R"                  S\R$                  S.rSrSr\R.                  " S\5      \R.                  " S	\5      S
 5       5       r\R.                  " S\5      \R.                  " S	\5      S 5       5       r\R.                  " S\5      \R.                  " S	\5      S 5       5       r\R.                  " S\5      \R.                  " S	\5      S 5       5       rSrg)TorchAOCompileTestCase   ri   r;   rj   rm   rq   2   r   r   c                 n   [         R                  " SSXS9nU R                  " U40 U R                  D6nS nU" U5      n[         R                  " U5      nU" U5      nU R                  [        U" U5      U R                  5      5        U R                  UR                  5       UR                  5       5        g )Nrs   rt   r   c                     U $ rL   r!   tensors    r   fCTorchAOCompileTestCase.test_input_output_tensor_subclass.<locals>.f  s    Mr    )
r)   ru   rv   r   compile
assertTrue
isinstancer{   r}   r~   rM   r   r   r   r   r   refcompileds           r   !test_input_output_tensor_subclass8TorchAOCompileTestCase.test_input_output_tensor_subclass  s     KK3vC	OOI==		 	lMM!Y<
1Y<1E1EFG)8+>+>+@Ar    c                 4   [         R                  " SSXS9nU R                  " U40 U R                  D6nS nU" U5      n[         R                  " U5      nU" U5      nU R                  [        U" U5      U R                  5      5        U R                  Xg5        g )Nrs   rt   r   c                 "    U R                  5       $ rL   )r~   r   s    r   r   <TorchAOCompileTestCase.test_input_tensor_subclass.<locals>.f  s    $$&&r    )	r)   ru   rv   r   r   assertFalser   r{   r}   r   s           r   test_input_tensor_subclass1TorchAOCompileTestCase.test_input_tensor_subclass  s     KK3vC	OOI==		' 	lMM!Y<AiL$2F2FGH'r    c                   ^  [         R                  " SSXS9nU 4S jnU" U5      n[         R                  " U5      nU" U5      nT R                  [	        U" U5      T R
                  5      5        U[         R                  :w  aa  T R                  [        R                  R                  R                  UR                  5       UR                  5       5      T R                  5        g g )Nrs   rt   r   c                 >   > TR                   " U 40 TR                  D6$ rL   )rv   r   )r   rM   s    r   r   =TorchAOCompileTestCase.test_output_tensor_subclass.<locals>.f(  s    ??9<<<r    )r)   ru   r   r   r   r{   r   r   r   r   r   r   r~   COMPILE_MIN_SQNR)rM   r   r   r   r   r   r   s   `      r   test_output_tensor_subclass2TorchAOCompileTestCase.test_output_tensor_subclass#  s     KK3vC		= 	lMM!Y<
1Y<1E1EFGENN"$$**88NN$h&9&9&; %%	 #r    c                 .   [         R                  " SSXS9nU R                  " U40 U R                  D6n[         R                  " SSXS9n[         R                  R
                  R                  XS5      n[         R                  R                  SSSXS9n[         R                  R                  U5      Ul	        [         R                  " U5      " U5      nU R                  [        R                  R                  R                  Xh5      U R                   5        g )Nrs   rt   r   rl   F)biasr   r   )r)   ru   rv   r   r   r   r   Linear	Parameterweightr   r   r   r   r   r   r   )	rM   r   r   r   r   r   r   r   r   s	            r   test_linear_compile*TorchAOCompileTestCase.test_linear_compile8  s     KK3vC	OOI==	BFH$$++MEaeFP**95v&}5  &&44VD  	
r    r!   N)r   r   r   r   r)   r;   r<   r   r   r   r   r   r   r{   r   rv   r   r   r   r   r   r   r   r   r   r   r   r   r   r!   r    r   r   r      sE   WEJJ,C,C,E,E2NN]]EMM5>>BM+O)J#..F Oh7g}5B 6 8B h7g}5( 6 8( h7g}5 6 8& h7g}5
 6 8
r    r   c                   Z   \ rS rSrSr\R                  \R                  \R                  /r	\
r\" \5      r0 r\S\R                   R"                  S\S\R                   R"                  4S j5       r\S\R                   R"                  S\S\R                   R"                  4S j5       rS\R                   R"                  S\R                   R"                  4S jr\R.                  " S	\	5      \\R4                  " \R6                  R9                  5       (       + S
5      S 5       5       5       rSrg)TorchAOTensorParallelTestCaseiI  z%Basic test case for tensor subclassesmmeshreturnc                 X   U R                   R                  nUR                  S5      UR                  5       -  nUR                  5       nX$U-  US-   U-  2SS24   n[        R
                  " XQ[        S5      /5      n[        R                  R                  USS9U R                   l        U $ )z8
Shard linear layer of the model in column-wise fashion
r   rk   NFrequires_grad
r   r   ry   get_local_rankr   
from_localr   r)   r   r   )r   r   orig_weightn_local_rowsranklocal_sharddtensors          r   colwise_shard+TorchAOTensorParallelTestCase.colwise_shardR  s     hhoo"''*diik9""$!"5\8Q"QST"TU$$[q
C((,,WE,Jr    c                 X   U R                   R                  nUR                  S5      UR                  5       -  nUR                  5       nUSS2XC-  US-   U-  24   n[        R
                  " XQ[        S5      /5      n[        R                  R                  USS9U R                   l        U $ )z5
Shard linear layer of the model in row-wise fashion
rk   NFr   r   )r   r   r   n_local_colsr   r   r   s          r   rowwise_shard+TorchAOTensorParallelTestCase.rowwise_shardc  s     hhoo"''*diik9""$!!T%8D1H;T%T"TU$$[q
C((,,WE,Jr    c                 P    [        XR                  " S0 U R                  D65        U$ )z
Quantize the model
r!   )r   QUANT_METHOD_FNQUANT_METHOD_KWARGS)rM   r   s     r   quantize&TorchAOTensorParallelTestCase.quantizet  s%     	!))ED,D,DEFr    r   zNeed CUDA availablec                    Sn[         R                  " S5         " S S[         R                  R                  5      n[         R                  " SU R
                  [         R                  R                  5       -   35      nU" SS5      R                  U5      R                  U5      nU" SS5      R                  U5      R                  U5      nS[         R                  " S	SX!S
9-  nU" U" U5      5        U R                  U5      nU R                  U5      nU" U" U5      5        U R                  5       n	SU	l        U R                  Xy5      n
U R                  X5      n[        R                   " Xi[#        5       /5      nU" U
" U5      5        [         R$                  " U
5      nU" U5      n[         R$                  " U5      nU" U5        g )Nr;      c                   f   ^  \ rS rSrSU 4S jjrS\R                  S\R                  4S jrSrU =r	$ )0TorchAOTensorParallelTestCase.test_tp.<locals>.Mi  r   c                 n   > [         TU ]  " S0 UD6  [        R                  R	                  XSSS9U l        g )NFr;   )r   r   r!   )super__init__r)   r   r   r   )rM   in_featuresout_featuresr   	__class__s       r   r   9TorchAOTensorParallelTestCase.test_tp.<locals>.M.__init__  s3     *6*#hhooE& . r    xc                 $    U R                  U5      $ rL   r   )rM   r  s     r   forward8TorchAOTensorParallelTestCase.test_tp.<locals>.M.forward  s    {{1~%r    r  )r   N)
r   r   r   r   r   r)   Tensorr  r   __classcell__)r   s   @r   Mr     s(    & &%,, & &r    r  zcuda:      d   rt   r   )r)   manual_seedr   Moduler   r   r;   device_countr   ru   r   build_device_meshdevice_typer   r   r   r   r   r   )rM   r   r   r  proj_upproj_dnexample_inputup_quantdn_quantr   up_distdn_distinput_dtensorup_compiledy_updn_compileds                   r   test_tp%TorchAOTensorParallelTestCase.test_tp{  sw    !	& 	& dii%**2I2I2K&K%LMN D$-""6*--e4D$-""6*--e4ekk#tFPP&' ==)==)-()%%'! $$X4$$X4  **=N&'mmG,=)mmG,Dr    r!   N)r   r   r   r   __doc__r)   r   r   r   r   r   r{   staticmethodr   r   r   r   r  r   r   r   r   r   r   r	   r   skipIfr;   r<   r  r   r!   r    r   r   r   I  s   /]]EMM5>>BM+O"#78O 
 uxx     
 uxx   %((// ehhoo  g}5__00224IJ, K  6,r    r   c                   D    \ rS rSrS\4S jrS rS\4S jrS\4S jrSr	g)	TorchAOIntegrationTestCasei  configc                 $   [         R                  nSn[         R                  R                  SSSUS9n[	        XA5        [         R                  R                  SS5      R                  S5      R                  [         R                  5      n[         R                  R                  UR                  S[         R                  " SSX2S9-  -   SS9Ul        [	        XQ5        SnSnS	 GH  nX-  n	UR                  n
U
R                  nUR                  XiU5      nUR                  S   S   nUR                  nUR                  XiU5      n[         R                  " XR                  S   S   5      (       a   eUR                  U5        [         R                  " UR                  S   S   UR                  S   S   5      (       d   e[        US
5      (       a2  [         R                  " UR                  UR                  5      (       d   e[        US5      (       a2  [         R                  " UR                   UR                   5      (       d   e[        US5      (       d  GMx  [         R                  " UR"                  UR"                  5      (       a  GM   e   g )Nr;   r	  r      Fr   r   i   )r   rk   scale
zero_pointscale_and_zero)r)   r   r   r   r   r   r   r   ru   datanarrowqdataequalcopy_rW   r&  r'  r(  )rM   r#  r   r   ldummy_l
output_dim
shard_sizetp_rank	start_idxparam
param_data
orig_valueloaded_weights                 r   $_test_slice_and_copy_similar_to_vllm?TorchAOIntegrationTestCase._test_slice_and_copy_similar_to_vllm  s    
 HHOOD$vUOC! ((//$-008;;ENNK++NNQT4!TTT , 
 	'"

G,IHHEJ#**:*MJ#))!,Q/J#NNM)00
SM {{:/B/B1/Ea/HIIII]+;;z//215}7J7J17Ma7PQQQQz7++{{:#3#3]5H5HIIIIz<00{{:#8#8-:R:RSSSSz#344{{--}/K/K   ' r    c                   ^ [         R                  nSnS nSnSnSnSnSn	[        XhXU5      n
[        XhXU5      n[        XhSU	-  X#5      n[         R                  " XWXUS9nX/nU GH8  nU" U5        [         R                  R                  UR                  R                  S	S5      R                  5       S
S9Ul        [         R                  R                  UR                  R                  S	S5      R                  5       S
S9Ul	        [         R                  R                  UR                  R                  S	S5      R                  5       S
S9Ul
        U" X5        U" U5      nUR                  R                  S   n[         R                  R                  UR                  R                  S	S5      R                  SU5      R                  5       S
S9Ul        UR                  R                  S   n[         R                  R                  UR                  R                  S	S5      R                  SU5      R                  5       S
S9Ul	        UR                  R                  S   n[         R                  R                  UR                  R                  S	S5      R                  SU5      R                  5       S
S9Ul
        [         R                  R                  UR                  R                  SUS45      R                  SS9S
S9Ul        [         R                  R                  UR                  R                  SUS45      R                  SS9S
S9Ul	        [         R                  R                  UR                  R                  SUS45      R                  SS9S
S9Ul
        [         R                  R                  UR                  R                  S	S5      R                  5       S
S9Ul        [         R                  R                  UR                  R                  S	S5      R                  5       S
S9Ul	        [         R                  R                  UR                  R                  S	S5      R                  5       S
S9Ul
        U" U5      nU R                  UU5        GM;     U
R!                  5       UR!                  5       /mS Hx  nT Vs/ s H  nUU   PM
     nn[#        S	[%        U5      5       HG  nUS   R&                  UU   l        [)        UU   S5      (       d  M0  US   R*                  UU   l        MI     Mz     S[,        S[         R.                  4U4S jjn0 nS H  nU" U5      UU'   M     [         R                  R                  UR                  R                  S	S5      S
S9Ul        [         R                  R                  UR                  R                  S	S5      S
S9Ul	        [         R                  R                  UR                  R                  S	S5      S
S9Ul
        UR1                  USS9  U" U5        gs  snf )zThis is testing the op call sequence in saving and loading quantization
checkpoints in llama-models for llama4
(https://github.com/meta-llama/llama-models/tree/main/models/llama4)
r;   c                    U R                  5        Hm  u  p#[        U[        5      (       d  M  UnS HI  n[        XE5      n[        [        U5         n[        U5      nU" X5      n	[        XEU	R                  5        MK     Mo     g )Nw1w2w3)	named_modulesr   r   rx   r   typer   r\   r   )
modelr#  rQ   moduleexpert_moduleweight_namer   config_handler	dummy_mod	quant_mods
             r   _quantize_expertsRTorchAOIntegrationTestCase._test_moe_weight_reshape_ops.<locals>._quantize_experts  ss    "002	!&*BCC &#5K$]@F%=d6l%KN +F 3I .y AIM	8H8HI $6 3r    rs   r%  @   rt      )r   r   rk   Fr   r   dimr<  r'  keyr   c                    > T Vs/ s H  oU    PM	     nnU S:X  a  [         R                  " USS9$ [         R                  " USS9$ s  snf )Nr>  rN  rO  rM  )r)   cat)rQ  stensorsstate_dictss      r   process_keyLTorchAOIntegrationTestCase._test_moe_weight_reshape_ops.<locals>.process_keyI  sI    '23{!v{G3 d{yyb11yyb11 4s   AT)assignN)r)   r   r   ru   r   r   r=  	transpose
contiguousr>  r?  r   reshape	unflattensqueezer}   
state_dictrangelenr&  rW   r'  strr  load_state_dict)rM   r#  r   r   rI  
batch_sizenum_experts	input_dimrP  
hidden_dimmoe1moe2moe_combinedinputmoesmoebeforenew_last_dimafterrQ  stweightsirW  new_state_dictrV  s                            @r   _test_moe_weight_reshape_ops7TorchAOIntegrationTestCase._test_moe_weight_reshape_ops  s    	J 
	
'*VT'*VT/a*ne
 J3FS|CJ XX''  A&1135 ( CF XX''  A&1135 ( CF XX''  A&1135 ( CF c*ZF 66<<+LXX''  A&..r<@KKM# ( CF 66<<+LXX''  A&..r<@KKM# ( CF 66<<+LXX''  A&..r<@KKM# ( CF
 XX''  ["$56>>1>E# ( CF XX''  ["$56>>1>E# ( CF XX''  ["$56>>1>E# ( CF XX''  A&1135 ( CF XX''  A&1135 ( CF XX''  A&1135 ( CF JEVU+y | ($//*;<%C)452r#wG51c'l+#*1:#3#3
 71:|44,3AJ,A,AGAJ) , &	2S 	2U\\ 	2 %C"-c"2N3 &  ((,,OO%%a+5 - 
  ((,,OO%%a+5 - 
  ((,,OO%%a+5 - 
 	$$^D$AU= 6s   Yc                    [         R                  n[         R                  R                  SSSUS9n[	        X15        UR
                  nUR                  SSS5      nUR                   Hf  n[        XF5      n[        XV5      n[        UR                  5      [        UR                  5      :X  a  MG   SUR                   SUR                   35       e   g )Nr	  r;   r   rk   r   zshape mismatch: z vs )r)   r   r   r   r   r   r*  tensor_data_namesrx   ra  r   )	rM   r#  r   r.  orignewdata_attr_name	orig_attrnew_attrs	            r   _test_narrow_similar_to_vllm7TorchAOIntegrationTestCase._test_narrow_similar_to_vllmc  s     HHOOD$vUOC!xxkk!Q%!33N5Is3Hy'3x~~+>> "9??"347GH> 4r    c           
      l   [         R                  n[         R                  " S5         [         R                  R	                  SSSUS9nS S S 5        [         R                  R                  [         R                  " SSSSUS95      Wl        [        X15        UR                  S   ng ! , (       d  f       Nc= f)	Nmetar	  r;   r   <   i   r
  r   )	r)   r   r   r   r   r   ru   r   r   )rM   r#  r   r.  _w_slices        r   '_test_quantize_3d_param_similar_to_vllmBTorchAOIntegrationTestCase._test_quantize_3d_param_similar_to_vllmt  s    
 \\&!d6GA "88%%KKD$vUC
 	!88A; "!s   !B%%
B3r!   N)
r   r   r   r   r
   r8  ru  r~  r  r   r!   r    r   r"  r"    s.    -< -^DL< "l r    r"  __main__rL   )NN)-rU   r3   r   r)   torch.distributed._tensorr   r   r   r   torch.testing._internalr   :torch.testing._internal.distributed._tensor.common_dtensorr   r	   r   torchao.core.configr
   torchao.dtypesr   r   torchao.quantizationr   r   %torchao.quantization.quant_primitivesr   %torchao.quantization.transform_moduler   #torchao.testing.model_architecturesr   torchao.utilsr   r   r%   r7   rA   rH   re   TestCaserg   r   r   r"  instantiate_parametrized_testsr   mainr!   r    r   <module>r     s        K K 0
  , J @ = I
2 8$=>M
<00 M
`Q
\22 Q
haO aHS!6!6 Sl  + +,@ A  + +,B C  + +,I J zMMO r    