
    ΅iΧ                    >   S r SSKrSSKJs  Js  Js  Js  Jr  SSK	Js  Js  Jr
  SSKJs  Js  Js  Jr  SSKJr  SSKJr  SSKJs  Jr  SSKJr  SSKJr  SSKJrJrJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-J.r.J/r/J0r0J1r1J2r2  SSK3J4r4J5r5J6r6  SSK7J8r8  SSK9J:r:  SS	K;J<r<J=r=   SS
K>J?r?J@r@  SSKAJBrBJCrCJDrDJErE  SSKFJGrG  SSKHJIrI  SrJSSKLrLSSKMrMSSKNrNSSKOrOSSKPrPSSKQJRrRJSrSJTrT  SSKUJVrV  SSKWrXSSKYJZr[  SSK\J]r]   " S S5      r^S r_S r`\R                  R                  5       rb\b4S jrc " S S5      rdSS jreS rfS rgS rhS riS rjSS  jrkS! rlS" rmS# rnS$ roS% rpS& rqS' rrS( rsS) rtS* ru SSKvrvSrw\PR                  " \w(       + S+5      ryS, rz\XR                  S4S- jr|SS. jr}SS/ jr~S0 r " S1 S2\=5      r " S3 S4\5      r " S5 S6\R                  GR                  5      r " S7 S8\R                  GR                  5      r " S9 S:\R                  GR                  5      r " S; S<\GR                  5      r " S= S>\R                  GR                  5      r " S? S@\R                  GR                  5      r " SA SB\R                  GR                  5      r " SC SD\R                  GR                  5      r " SE SF\R                  GR                  5      r " SG SH\R                  GR                  5      r " SI SJ\R                  GR                  5      r " SK SL\R                  GR                  5      r " SM SN\R                  GR                  5      r " SO SP\R                  GR                  5      r " SQ SR\R                  GR                  5      r " SS ST\R                  GR                  5      r " SU SV\R                  GR                  5      r " SW SX\GR                  5      r " SY SZ\R                  GR                  5      r " S[ S\\R                  GR                  5      r " S] S^\R                  GR                  5      r " S_ S`\R                  GR                  5      r " Sa Sb\R                  GR                  5      r " Sc Sd\R                  GR                  5      r " Se Sf\R                  GR                  5      r " Sg Sh\R                  GR                  5      r " Si Sj\R                  GR                  5      r " Sk Sl\R                  GR                  5      r " Sm Sn\R                  GR                  5      r " So Sp\R                  GR                  5      r " Sq Sr\R                  GR                  5      r " Ss St\R                  GR                  5      r " Su Sv\R                  GR                  5      r " Sw Sx\R                  GR                  5      r " Sy Sz\R                  GR                  5      r " S{ S|\R                  GR                  5      r " S} S~\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\GR                  5      r " S S\GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\GR                  5      r " S S\GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\5      r " S S\GR                  5      r " S S\GR                  5      r " S S\GR                  5      r " S S\GR                  5      r " S S\GR                  5      r " S S\GR~                  5      r " S S\GR                  5      r " S S\GR                  5      r " S S\GR                  5      r " S S\R                  GR                  5      r " S S\GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\R                  GR                  5      r " S S\GR                  5      r " S S\GR                  5      r " S S\GR                  5      r " S S5      rg! \K a    SrJ G	Nf = f! \K a    Srw G	N!f = f)zImporting this file includes common utility methods and base classes for
checking quantization api and properties of resulting modules.
    N)control_flow)_FusedModule)convertdefault_dynamic_qat_qconfigdefault_dynamic_qconfigdefault_dynamic_quant_observerdefault_embedding_qat_qconfigdefault_observerdefault_per_channel_qconfigdefault_qconfig%default_symmetric_qnnpack_qat_qconfigdefault_weight_observerDeQuantStub!float_qparams_weight_only_qconfigget_default_qat_qconfigget_default_qat_qconfig_mappingget_default_qconfigget_default_qconfig_mappingPerChannelMinMaxObserverpropagate_qconfig_QConfigQConfigMappingquantizequantize_dynamic_jitquantize_jit	QuantStub	QuantTypeQuantWrapper))get_default_dynamic_quant_module_mappingsget_default_qat_module_mappings$get_default_qconfig_propagation_list)_load_for_lite_interpreter)override_quantized_engine)TEST_WITH_ROCMTestCase)NSSingleResultValuesType
NSSubgraph)
convert_fxconvert_to_reference_fx
prepare_fxprepare_qat_fx)GraphModule)NodeTF)AnyOptionalUnion)Callable)	FileCheckc                   `    \ rS rSrSrS r\S 5       r\S 5       r\S 5       r	S r
S rS	 rS
rg)NodeSpecU   z"Used for checking GraphModule Nodec                     Xl         X l        g)z
op: call_function | call_module
target:
  for call_function, target would be a function
  for call_module, target would be the type of PyTorch module
Noptarget)selfr8   r9   s      e/home/james-whalen/.local/lib/python3.13/site-packages/torch/testing/_internal/common_quantization.py__init__NodeSpec.__init__X   s         c                     [        SU5      $ )Ncall_functionr4   clsr9   s     r;   r@   NodeSpec.call_functionb   s    00r>   c                     [        SU5      $ )Ncall_methodrA   rB   s     r;   rF   NodeSpec.call_methodf       v..r>   c                     [        SU5      $ )Ncall_modulerA   rB   s     r;   rJ   NodeSpec.call_modulej   rH   r>   c                 D    [        U R                  U R                  45      $ N)hashr8   r9   r:   s    r;   __hash__NodeSpec.__hash__n   s    TWWdkk*++r>   c                     [        U[        5      (       d  [        $ U R                  UR                  :H  =(       a    U R                  UR                  :H  $ rM   )
isinstancer4   NotImplementedr8   r9   )r:   others     r;   __eq__NodeSpec.__eq__q   s:    %**!!ww%(("Bt{{ell'BBr>   c                 ^    [        U R                  5      S-   [        U R                  5      -   $ )N )reprr8   r9   rO   s    r;   __repr__NodeSpec.__repr__w   s#    DGG}s"T$++%666r>   r7   N)__name__
__module____qualname____firstlineno____doc__r<   classmethodr@   rF   rJ   rP   rV   r[   __static_attributes__ r>   r;   r4   r4   U   sV    , 1 1 / / / /,C7r>   r4   c                  l    [         R                  R                  5       (       a  [        (       d  SS/$ S/$ )Ncpucuda)torchrg   is_availabler$   rd   r>   r;   get_supported_device_typesrj   {   s/     ::2244^^RWQXr>   c                      U H  nU " U6   M
     g)zz
Default evaluation function takes a torch.utils.data.Dataset or a list of
input Tensors and run the model on the dataset
Nrd   )model
calib_datainps      r;   test_only_eval_fnro      s    
 s r>   c                    [         R                  R                  U R                  5       SS9nSu  pEn[	        S5       H  nU R                  5         U H  u  pUR                  5         U " U5      n
U" X5      nUR                  5         UR                  5         XKR                  5       -  n[         R                  " U
S5      u  p|XiR                  S5      -  nX\U	:H  R                  5       R                  5       -  nM     M     XEU4$ )z\
Default train function takes a torch.utils.data.Dataset and train the model
on the dataset
gMbP?lr)r   r   r   
      r   )rh   optimAdam
parametersrangetrain	zero_gradbackwardstepitemmaxsizesum)rl   
train_dataloss_fn	optimizer
train_losscorrecttotal_datar9   outputloss	predicteds                r;   test_only_train_fnr      s    
   !1!1!3 >I!(J2Y&LD!4[F6*DMMONN))+%J 99VQ/LA[[^#EV+0027799G '  %%r>   c                   8    \ rS rSrSrS	S jrS rS
S jrS rSr	g)AverageMeter   z1Computes and stores the average and current valuec                 <    Xl         X l        U R                  5         g rM   )namefmtreset)r:   r   r   s      r;   r<   AverageMeter.__init__   s    	

r>   c                 <    SU l         SU l        SU l        SU l        g Nr   )valavgr   countrO   s    r;   r   AverageMeter.reset   s    
r>   c                     Xl         U =R                  X-  -  sl        U =R                  U-  sl        U R                  U R                  -  U l        g rM   )r   r   r   r   )r:   r   ns      r;   updateAverageMeter.update   s8    CG

a
88djj(r>   c                 ~    SU R                   -   S-   U R                   -   S-   nUR                  " S0 U R                  D6$ )Nz{name} {valz} ({avgz})rd   )r   format__dict__)r:   fmtstrs     r;   __str__AverageMeter.__str__   s9    )I5@4G}}-t}}--r>   )r   r   r   r   r   r   N)z:frt   )
r]   r^   r_   r`   ra   r<   r   r   r   rc   rd   r>   r;   r   r      s    ;
).r>   r   c                    [         R                  " 5          [        U5      nUR                  S5      nU R	                  USSS5      u  pVUR                  5       nUR                  UR                  SS5      R                  U5      5      n/ nU HV  n	USU	 R                  S5      R                  5       R                  SSS9n
UR                  U
R                  SU-  5      5        MX     UsSSS5        $ ! , (       d  f       g= f)zNComputes the accuracy over the k top predictions for the specified values of kr   rt   TN)keepdimg      Y@)rh   no_gradr~   r   topkteqview	expand_asfloatr   appendmul_)r   r9   r   maxk
batch_sizer   predr   resk	correct_ks              r;   accuracyr      s    	4y[[^
++dAtT2vvx''&++a,66t<=A((,22488D8IIJJy~~ej&89:   
s   CC22
D c                 H   U R                  5         [        USS9 H  u  nu  px[        SSS9  UR                  U5      UR                  U5      pU " U5      n	U" X5      n
UR	                  5         U
R                  5         UR                  5         [        XSS9  Xe:  d  M    g    g )Nrt   )start. )end)rt      )r   )ry   	enumerateprinttorz   r{   r|   r   )rl   	criterionr   data_loaderdevicentrain_batchescntimager9   r   r   s              r;   train_one_epochr      s    	KKM )+Q ?_ecr(&))F*;vu(f-  !@ r>   c                 z    S[         R                  S'   S[         R                  S'   [        R                  " SXS9  g )N	localhostMASTER_ADDR12355MASTER_PORTgloorank
world_size)osenvirondistinit_process_groupr   s     r;   	ddp_setupr      s0     +BJJ} 'BJJ} 	FEr>   c                  .    [         R                  " 5         g rM   )r   destroy_process_grouprd   r>   r;   ddp_cleanupr      s     r>   c                 L   [        X5        UR                  5         [        R                  R                  R                  X /S9nUR                  U 5        Un[        R                  R                  UR                  5       SS9n[        U[        U[        U S5        [        5         g )N)
device_idsg-C6?rq   rt   )r   rg   rh   nnparallelDistributedDataParallelr   ru   SGDrw   r   r   datasetr   )r   r   preparedmodel_with_ddpr   s        r;   run_ddpr      sy    dMMOxx  88f8UHKKN 9 9 ;GINIy'4KMr>   c                 *    [        U [        5       SS9  g )NTinplace)r   r   )modules    r;   convert_dynamicr      s    F=?Nr>   c                     [        X5        g rM   )r   )rl   qconfig_dicts     r;   prepare_dynamicr      s
    u+r>   c                 ,   X-  nX4-  nSu  p[         R                  " UUU U4U-   5      nUUU-
  R                  5       -  n[         R                  " UXg[         R                  S9nX-  nX-  n	US U nU	S U n	Su  nn[         R                  " UUUU4U-   5      n[         R                  " SSU45      nU(       a  SS[        U5      -  -   n[         R                  " U[         R                  S9n[         R                  " U	[         R                  S9nUR                  " U6 UR                  5       UR                  " U6 -
  R                  5       -  nUU-  UR                  5       -  n[         R                  " UUR                  5       UR                  5       S[         R                  S9nO_US   UU	S   -
  R                  5       -  nXhS   -  UR                  5       -  n[         R                  " UUS   U	S   [         R                  S9nUUUUU
(       a  U4$ S 4$ )	Nr      scale
zero_pointdtype)r   r   rs   )r   rt   r   r   )rh   randintr   quantize_per_tensorquint8lentensorreshapequantize_per_channeldoublelongqint8)r   in_channels_per_groupinput_feature_map_sizeout_channels_per_groupgroupskernel_sizeX_scaleX_zero_pointW_scaleW_zero_pointuse_biasuse_channelwisein_channelsout_channelsX_value_minX_value_maxX_initXX_qW_value_minW_value_maxW_initb_initW_shapeW_scales_tensorW_zero_points_tensorWbW_qs                                r;   _make_conv_test_inputr     s4    (0K)2L!'[]]	
 !		!F 	6L(//11A

#
#	C $G.Lm|$G.L ")[+ ]]!	
 		F ]]1b</2FD3{#333,,wekkB$||LL##W-||~ 4 < <g FFMMOP 	
 o%6((""$ %%'++
 AJ&<?299;;aj 6<<>1''WQZLO5;;
 sAsA44t44r>   c                     Su  p4[         R                  " UUU5      nXU-
  R                  5       -  n[         R                  " X`U[         R                  S9nXg4$ )Nr   r   )rh   r   r   r   r   )r   r   sizesr  r  r  r  r  s           r;   !_make_conv_add_extra_input_tensorr  P  s]    !'[]]F
 	*$++--A

#
#	:U\\C 6Mr>   c                    ^ ^ Sm[        T [        5      (       a8  S[        R                  R                  R
                  ;  a  ST l        TT l        T $ [        R                  " T 5      U U4S j5       nU$ )NzrQuantized operations require FBGEMM. FBGEMM is only optimized for CPUs with instruction set support AVX2 or newer.fbgemmTc                     > S[         R                  R                  R                  ;  a  [        R
                  " T5      eT" U 0 UD6  g )Nr   rh   backends	quantizedsupported_enginesunittestSkipTestargskwargsfnreasons     r;   wrapperskipIfNoFBGEMM.<locals>.wrapperf  9    5>>33EEE##F++r>   
rS   typerh   r#  r$  r%  __unittest_skip____unittest_skip_why__	functoolswrapsr+  r-  r,  s   ` @r;   skipIfNoFBGEMMr7  ^  sg     BF"d5>>33EEE#'B '-B$	__R    Nr>   c                    ^ ^ Sm[        T [        5      (       a8  S[        R                  R                  R
                  ;  a  ST l        TT l        T $ [        R                  " T 5      U U4S j5       nU$ )N%Quantized operations require QNNPACK.qnnpackTc                     > S[         R                  R                  R                  ;  a  [        R
                  " T5      eT" U 0 UD6  g Nr:  r"  r(  s     r;   r-   skipIfNoQNNPACK.<locals>.wrapperx  s9    ENN44FFF##F++r>   r0  r6  s   ` @r;   skipIfNoQNNPACKr>  p  sd    4F"dENN44FFF#'B '-B$	__R    Nr>   c                    ^ ^ Sm[        T [        5      (       a8  S[        R                  R                  R
                  ;  a  ST l        TT l        T $ [        R                  " T 5      U U4S j5       nU$ )Nr9  r:  Tc                     > S[         R                  R                  R                  ;  a  [        R
                  " T5      e[        S5         T" U 0 UD6  S S S 5        g ! , (       d  f       g = fr<  )rh   r#  r$  r%  r&  r'  r#   r(  s     r;   r-  #withQNNPACKBackend.<locals>.wrapper  sM    ENN44FFF##F++&y1 211s   	A
A+r0  r6  s   ` @r;   withQNNPACKBackendrB    sf     5F"dENN44FFF#'B '-B$	__R    Nr>   c                    ^ ^ Sm[        T [        5      (       a8  S[        R                  R                  R
                  ;  a  ST l        TT l        T $ [        R                  " T 5      U U4S j5       nU$ )Nz$Quantized operations require ONEDNN.onednnTc                     > S[         R                  R                  R                  ;  a  [        R
                  " T5      eT" U 0 UD6  g )NrD  r"  r(  s     r;   r-  skipIfNoONEDNN.<locals>.wrapper  r/  r>   r0  r6  s   ` @r;   skipIfNoONEDNNrG    sd    3F"d5>>33EEE#'B '-B$	__R    Nr>   c                    ^ ^ Sm[        T [        5      (       a=  [        R                  R                  R                  5       (       d  ST l        TT l        T $ [        R                  " T 5      U U4S j5       nU$ )Nz*Quantized operations require BF16 support.Tc                     > [         R                  R                  R                  5       (       d  [        R
                  " T5      eT" U 0 UD6  g rM   )rh   opsmkldnn_is_mkldnn_bf16_supportedr&  r'  r(  s     r;   r-  #skipIfNoONEDNNBF16.<locals>.wrapper  s:    yy99;;##F++r>   )
rS   r1  rh   rJ  rK  rL  r2  r3  r4  r5  r6  s   ` @r;   skipIfNoONEDNNBF16rN    se    9F"dyy99;;#'B '-B$	__R    Nr>   c                    ^ ^ Sm[        T [        5      (       a8  S[        R                  R                  R
                  ;  a  ST l        TT l        T $ [        R                  " T 5      U U4S j5       nU$ )Nz!Quantized operations require X86.x86Tc                     > S[         R                  R                  R                  ;  a  [        R
                  " T5      eT" U 0 UD6  g )NrP  r"  r(  s     r;   r-  skipIfNoX86.<locals>.wrapper  s9    00BBB##F++r>   r0  r6  s   ` @r;   skipIfNoX86rS    sd    0F"d00BBB#'B '-B$	__R    Nr>   c                    ^ ^ Sm[        T [        5      (       a*  [        R                  " 5       (       d  ST l        TT l        T $ [        R                  " T 5      U U4S j5       nU$ )Nzdynamo doesn't support.Tc                  v   > [         R                  " 5       (       d  [        R                  " T5      eT" U 0 UD6  g rM   )torchdynamois_dynamo_supportedr&  r'  r(  s     r;   r-  &skipIfNoDynamoSupport.<locals>.wrapper  s0    ..00##F++r>   )rS   r1  rV  rW  r2  r3  r4  r5  r6  s   ` @r;   skipIfNoDynamoSupportrY    s[    &F"d..00#'B '-B$	__R    Nr>   c                    ^ ^ Sm[        T [        5      (       a*  [        R                  " 5       (       d  ST l        TT l        T $ [        R                  " T 5      U U4S j5       nU$ )Nzinductor doesn't support.Tc                  v   > [         R                  " 5       (       d  [        R                  " T5      eT" U 0 UD6  g rM   )rV  is_inductor_supportedr&  r'  r(  s     r;   r-  (skipIfNoInductorSupport.<locals>.wrapper  s0    0022##F++r>   )rS   r1  rV  r\  r2  r3  r4  r5  r6  s   ` @r;   skipIfNoInductorSupportr^    s[    (F"d0022#'B '-B$	__R    Nr>   zno torchvisionc                     U(       a  [         R                  R                  X5      $ [         R                  R                  U 5      $ rM   )rh   jittracescript)rl   tracingr   s      r;   get_script_modulerd    s+    +2599??5'O		8H8H8OOr>   c                     [         R                  " U R                  S   S-   4US9nXSS& [        R                  " [         R
                  " X1S95      nU(       a  USS $ USS $ )z.
Convert lengths to offsets for embedding_bag
r   rt   r   Nr   )npzerosshaperh   
from_numpycumsum)r   offset_typeuse_begin_offsettts       r;   lengths_to_offsetsrn    s^     
1771:>#;	7BqrF			"))B:	;B#2wab6Mr>   c           	         U R                  5       S:X  d   eU R                  SS5      R                  5       n US:  d   eU R                  S   U-  S:X  d   eU R	                  SU5      n[
        R                  " U5      R                  5       S:X  d   eUR                  SSS9nUR                  SSS9nSU-  S-
  nSnXE-
  R                  SS9U-  n[
        R                  " U5      R                  5       S:X  d   eXXSUS-
  -  -  -   n	[
        R                  " U	5      R                  5       S:X  d   eUR                  U5      R                  U5      R                  5       R                  Xv5      n
[
        R                  " U
5      R                  5       S:X  d   eU
R                  [
        R                   S	9R	                  U R                  5      n
U
R"                  [
        R"                  " S
5      :w  a9  U
S S 2S S S24   S-  U
S S 2SS S24   -  R                  [
        R$                  5      n
UR'                  U R                  S   S5      nU	R'                  U R                  S   S5      n	[
        R(                  " UR	                  UR+                  S5      UR+                  S5      S5      U	R	                  U	R+                  S5      U	R+                  S5      S5      /S5      R                  SS5      R                  5       nX4$ )N   r   rt   r   Tdimr   gư>minr   rf   r   )rr  	transpose
contiguousrh  r   rh   isnanr   amaxaminclampsubdivroundclamp_r   int32r   uint8r   catr   )wn_bitq_group_sizeto_quantmax_valmin_valmax_intmin_intscalesrg  outscales_and_zeross               r;   _group_quantize_tensorr  	  s   557a<<	Aq$$&A!772;%***yy\*H;;x $$&!+++mm4m0Gmm4m0GhlGG&&4&07:F;;v""$)))eai 011E;;u!!#q(((
,,w

#
#F
+
1
1
3
:
:7
LC;;s!Q&&&
&&u{{&
#
+
+AGG
4C
zzU\\%((2ss7|q 3r14a4x=044U[[A [[R(FJJqwwqz2&E		v{{1~v{{1~qAejjmUZZ]A> 	
 
1a	    r>   c                 X   U R                  SS5      R                  5       n U R                  5       S:X  d   eUS:  d   eU R                  S   U-  S:X  d   eU R	                  SU5      nUR                  5       R                  SSS9n[        R                  " UR                  5      R                  nSUS-
  -  S-
  nUR                  US9U-  n[        R                  " U5      nUR                  [        R                  5      R	                  U R                  S   S5      nUR                  [        R                  5      R	                  U R                  S   S5      nUR	                  SS5      nUR	                  SS5      nSU-  S-
  nUR                  U5      R!                  S5      R                  [        R"                  5      R                  US	9n	U	S S 2SS S24   S
-  U	S S 2S S S24   -  R                  [        R$                  5      n
UR'                  5       R                  5       nX4$ )Nr   rt   rp  r   Trq  rs  g      !@)r~   r   )ru  rv  rr  rh  r   absrx  rh   finfor   epsrz  
zeros_liker   float32r|  addint8r  squeeze)r  r  	groupsizer  r  r  r  r  rg  w_int8	out_uint8r  s               r;    _group_quantize_tensor_symmetricr  6  s    	
Aq$$&A557a<<q==772;"a'''yyY'Hlln!!a!6G
++gmm
$
(
(CEAI"G]]s]#g-FV$E YYu}}%--aggaj"=FHHU]]#++AGGAJ;E^^B"FMM"a EhlG\\&!%%c*--ejj9??G?LF ADqD!Q&CaC8<<U[[II~~'224&&r>   c                    U R                   nU R                  5       n [        R                  " [        R                  5      R
                  n[        R                  " U SS9u  pg[        R                  " U[        R                  " U5      5      n[        R                  " U[        R                  " U5      5      n	UR                  n
[        R                  " U* U	5      n	U	[        X!-
  5      S-  -  n[        R                  " XS9R                  U R                   5      n[        R                  " UR                  5       [        R                  U
S9nXR!                  S5      -  n[        R"                  " U5      nXR!                  S5      -   n[        R                  " XU5      R                  U5      nUUR                  U5      U4$ )Nrt   rr  rp  rs  )r   r   r   )r   r   rh   r  r  r  aminmaxrt  r  r~   r   rz  r   rg  r   int64	unsqueezer}  )x	quant_min	quant_maxtarget_dtypex_dtyper  r  r  min_val_negmax_val_posr   r  zero_pointsx_divx_roundx_zpquants                    r;   !_dynamically_quantize_per_channelr  U  sV    ggG		A
++emm
$
(
(C }}QA.G ))GU%5%5g%>?K))GU%5%5g%>?KF ))[L+6KE)"781<=F[[),,QWW5F++k..0FSK   $$Ekk% G**2..DKK366|DE&))G$k11r>   c            
         ^  \ rS rSrU 4S jrS rS rS r S#S jrS r	S	 r
S
 rS rS rS rS rS rS rS rS$S jrS rS r      S%S jr   S&S jrS'S jr\(       aU  S\\\\\4   4   S\\\\\\4   \\\4   4   4   S\S\SS4
S jr S\\\\\\\!4   4   4   SS4S jr"           S(S  jr#\$RJ                  4S! jr&S"r'U =r($ ))QuantizationTestCaseiv  c                   > [         TU ]  5         [        S5       Vs/ s H(  n[        R                  " SS[        R
                  S9/PM*     snU l        [        S5       Vs/ s HL  n[        R                  " SS[        R
                  S9[        R                  " SSS[        R                  S9/PMN     snU l	        [        S5       Vs/ s H)  n[        R                  " SSS[        R
                  S9/PM+     snU l
        [        S5       Vs/ s H*  n[        R                  " SSSS[        R
                  S9/PM,     snU l        [        S5       Vs/ s H+  n[        R                  " SSSSS[        R
                  S9/PM-     snU l        [        S5       Vs/ s HM  n[        R                  " SSS[        R
                  S9[        R                  " SSS	[        R                  S9/PMO     snU l        [        S5       Vs/ s HN  n[        R                  " SSSS[        R
                  S9[        R                  " SSS	[        R                  S9/PMP     snU l        [        S5       Vs/ s HO  n[        R                  " SSSSS[        R
                  S9[        R                  " SSS	[        R                  S9/PMQ     snU l        U R                  U R                  U R                  S
.U l        ["        R$                  ["        R&                  /U l        ["        R*                  ["        R$                  ["        R&                  /U l        g s  snf s  snf s  snf s  snf s  snf s  snf s  snf s  snf )Nrp  r   r   r   rt   )rp     rs   r   rt   rp  r  )supersetUprx   rh   randr   rm   r   r   r   img_data_1dimg_data_2dimg_data_3dimg_data_1d_trainimg_data_2d_trainimg_data_3d_trainimg_data_dictr   STATICQATstatic_quant_typesDYNAMICall_quant_types)r:   r   	__class__s     r;   r  QuantizationTestCase.setUpw  s   JOPQ(S(QEJJq!5;;?@(S 1X

  

1au{{3aD

; 
 PUUVxXx!UZZ1bDExXCH8
CKaUZZ1b"EKK898
 EJ!H
DLqUZZ1aAU[[9:H
 1X"

  

1a5;;7aD

; "
 1X"

  

1aRu{{;aD

; "
 1X"

  

1aAq<aD

; "
 
 $-#3#3Y]]"C ) 1 193C3CY]]S] T
 Y

"
"
"
s3   /M!AM	0M1M2M#AM(AM-.AM2c                 p    U R                  [        US5      5        U R                  [        US5      5        g)zpChecks the module does not contain child
modules for quantization preparation, e.g.
quant, dequant and observer
r  dequantN)assertFalsehasattrr:   r   s     r;   checkNoPrepModules'QuantizationTestCase.checkNoPrepModules  s.    
 	1234r>   c                     U R                  [        US5      5        UR                  5        H  nU R                  U5        M     g)z*Checks the module does not contain qconfigqconfigN)r  r  childrencheckNoQconfig)r:   r   childs      r;   r  #QuantizationTestCase.checkNoQconfig  s6    34__&E& 'r>   c                     U R                  [        US5      5        U R                  [        US5      5        U R                  [        US5      5        g)zhChecks the module contains child
modules for quantization preparation, e.g.
quant, dequant and observer
r   r  r  N)
assertTruer  r  s     r;   checkHasPrepModules(QuantizationTestCase.checkHasPrepModules  s=    
 	1201	23r>   Nc           	      F   Uc
  [        5       nUc  0 nUR                  S0 5      nS n[        US5      (       a  UR                  b  U" U5      (       a8  [	        U[
        R                  R                  5      (       d  [        U5      U;   d  [        U5      U;   ag  [	        U[
        R                  R                  R                  5      (       d4  U R                  [        US5      S[        [        U5      5      -   S-   5        [        U5      [        5       R                  5       ;  a|  [        U5      UR                  5       ;  a^  [	        U[         5      (       dH  UR#                  5        H3  n[        U5      [        R$                  L a  M!  U R'                  XbU5        M5     gggg)z^Checks the module or module's leaf descendants
have observers in preparation for quantization
N%float_to_observed_custom_module_classc                 \    SnU R                  5        H  u  p#US:w  d  M  US-  nM     US:H  $ )Nr   activation_post_processrt   )named_children)r   submodule_name_countr   r   s       r;   is_leaf_module;QuantizationTestCase.checkObservers.<locals>.is_leaf_module  s>    #$ !00244(A-( 3 (1,,r>   r  r  zmodule: z do not have observer)r!   getr  r  rS   rh   r   
Sequentialr1  aoquantizationr   r  strr    valuesr   r  DropoutcheckObservers)r:   r   propagate_qconfig_listprepare_custom_config_dict&float_to_observed_module_class_mappingr  r  s          r;   r  #QuantizationTestCase.checkObservers  sc    ")%I%K"%-)+&1K1O1O3R2
.
	- FI&&* #6**&vuxx/B/BCCV(>><#IIvuxx'<'<'H'HIIOO 9:Sf..1HH L ? A H H JJV$J$Q$Q$SSv|44*;"**,##3M + 5 T Kr>   c                     U R                  [        UR                  5      [        R                  5        U R                  [        UR
                  5      [        R                  5        g)zFChecks that mod has nn.Quantize and
nn.DeQuantize submodules inserted
N)assertEqualr1  r  nnqQuantizer  
DeQuantizer:   mods     r;   checkQuantDequant&QuantizationTestCase.checkQuantDequant  s=     	cii#,,7ckk*CNN;r>   c                     U R                  [        UR                  5      [        R                  5        U R                  U5        g)zChecks that mod has been swapped for an nnq.Linear
module, the bias is qint32, and that the module
has Quantize and DeQuantize submodules
N)r  r1  r   r  Linearr  r  s     r;   checkWrappedQuantizedLinear0QuantizationTestCase.checkWrappedQuantizedLinear  s/    
 	cjj)3::6s#r>   c                 V    U R                  [        U5      [        R                  5        g rM   )r  r1  r  r  r  s     r;   checkQuantizedLinear)QuantizationTestCase.checkQuantizedLinear  s    cCJJ/r>   c                     U R                  [        U5      [        R                  5        U R                  UR                  R
                  U5        gOChecks that mod has been swapped for an nnqd.Linear
module, the bias is float.
N)r  r1  nnqdr  _packed_paramsr   r:   r  r   s      r;   checkDynamicQuantizedLinear0QuantizationTestCase.checkDynamicQuantizedLinear  s8     	cDKK0++1159r>   c                     U R                  [        U5      [        R                  5        U R                  UR                  R
                  U5        gr  )r  r1  nniqd
LinearReLUr  r   r  s      r;   checkDynamicQuantizedLinearRelu4QuantizationTestCase.checkDynamicQuantizedLinearRelu  s:     	cE$4$45++1159r>   c                   ^  UR                  5       n[        R                  " 5       n[        R                  " XE5        UR                  S5        [        R                  " USS9nUR                  U5        U" U6 nU" U6 nU 4S jn	U	" Xx5        [        R                  " 5       n[        R                  " X5        UR                  S5        [        R                  " USS9n
U
" U6 nU	" Xx5        g )Nr   F)weights_onlyc                   > TR                  U S   US   5        [        U S   [        5      (       a=  TR                  U S   S   US   S   5        TR                  U S   S   US   S   5        g TR                  U S   US   5        g )Nr   rt   )r  rS   tuple)ref_outload_outr:   s     r;   check_outputsEQuantizationTestCase.check_eager_serialization.<locals>.check_outputs"  s    WQZ!5'!*e,,  AA?  AA?  Xa[9r>   )
state_dictioBytesIOrh   saveseekloadload_state_dict)r:   	ref_modelloaded_modelr  
model_dictr  loaded_dictr	  r
  r  loadeds   `          r;   check_eager_serialization.QuantizationTestCase.check_eager_serialization  s    ))+
JJL

:!	q	jj7$$[1Q-#	: 	g(JJL

9 	q	AE21:g(r>   c                     UR                  5       nUR                  5       nU R                  X$R                  5       -  [	        5       5        U R                  X5R                  5       -  [	        5       5        g rM   )
get_weightget_biasr  keysset)r:   r  weight_keys	bias_keysweightbiass         r;   check_weight_bias_api*QuantizationTestCase.check_weight_bias_api3  sT    %%'!!#{{}4ce<YY[0#%8r>   c                    [         R                  S[         R                  S0nU R                  [	        U5      U5        UR
                   H5  nU R                  UR                  R                  5       S   S   XC   5        M7     g)zRChecks that mod has been swapped for an nnqd.LSTM type
module, the bias is float.
quantized_dynamicquantized_fp16r   N)rh   r   float16r  r1  _all_weight_valuesparam__getstate__r:   r  reference_module_typer   wt_dtype_mappacked_paramss         r;   checkDynamicQuantizedLSTM.QuantizationTestCase.checkDynamicQuantizedLSTM9  su    
 KK,MM+
 	c$9: 33M##00215a8,:M 4r>   c                 j    U R                  [        U5      [        R                  R                  5        g rM   )r  r1  rh   r   r  r  s     r;   checkLinear QuantizationTestCase.checkLinearG  s    cEHHOO4r>   c                 ,   [         R                  S[         R                  S0nU R                  [	        U5      U5        [        US5      (       aF  UR                   H5  nU R                  UR                  R                  5       S   S   XC   5        M7     gg)r  r'  r(  r*  r   N)	rh   r   r)  r  r1  r  r*  r+  r,  r-  s         r;   checkDynamicQuantizedModule0QuantizationTestCase.checkDynamicQuantizedModuleJ  s    
 KK,MM+
 	c$9:3,--!$!7!7  !''446q9!<l>Q "8 .r>   c                     [         R                  R                  U5      nU R                  XX#5        [         R                  R	                  XS   5      nU R                  XX#5        g r   )rh   r`  rb  _checkScriptablera  )r:   orig_modrm   check_save_loadscriptedtraceds         r;   checkScriptable$QuantizationTestCase.checkScriptableY  sN    99##H-h*N a=9h
Lr>   c                 $   U R                  XU5        [        R                  " 5       n[        R                  R                  X%5        UR                  S5        [        R                  R                  U5      nU(       a  U R                  XU5        g g r   )"_checkModuleCorrectnessAgainstOrigr  r  rh   r`  r  r  r  )r:   r;  
script_modrm   r<  buffer
loaded_mods          r;   r:  %QuantizationTestCase._checkScriptableb  sf    //jQ 		z*AYY^^F+
 33H*U r>   c                 L    U H  nU" U6 nU" U6 nU R                  Xe5        M      g rM   )r  )r:   r;  test_modrm   rn   
ref_outputscripted_outputs          r;   rB  7QuantizationTestCase._checkModuleCorrectnessAgainstOrigp  s,    C!3J&nO_9 r>   c
           
      P   U(       a  [        S[        U5      5        S[        [        R                  R
                  R                  5      0n
U(       a  UR                  5       nU(       a  SU	c  [        OU	0n
[        XUS   5      R                  5       nU(       a  [        SUR                  5        0 n0 nS H[  nU(       a  [        XUS9X'   X   " U5      X'   M$  [        R                  " U5      n[        UU
[        U/SUS9X'   X   " US   6 X'   M]     U(       a2  [        S	US
   R                  5        [        SUS   R                  5        U(       aM  U R!                  US
   US   5        [#        5       R%                  U5      R'                  US   R                  5        US   $ )NzTesting:r   r   zinput graph:)TF)debugF)r   rM  zdebug graph:Tznon debug graph:)r   r  r   rh   r#  r$  engineevalr   rd  graphr   copydeepcopyr   ro   r  r2   checkrun)r:   r   inputsquantized_oprc  rM  rS  	eval_modedynamicr  r   rl   modelsoutputsinputs_copys                  r;   checkGraphModeOp%QuantizationTestCase.checkGraphModeOpv  st    *c&k*/0H0H0O0OPQ[[]F7? 7PWXL!&6!9=BBD.%++."E 4UPU V!'v!6 #mmF3 , % M!! "(q	!:% #( .&,"4"45$fUm&9&9:WT]GEN; Kl+//u0C0CDe}r>   c           	      L   0 n/ n[        UR                  SS95      nUR                  R                   H  nSn	UR                  S:X  d  UR                  S:X  a!  [        UR                  UR                  5      n	O;UR                  S:X  a+  [        UR                  [        XxR                     5      5      n	U	c  M  UR                  U	5        X;   a  XY==   S-  ss'   M  SXY'   M     Ub#  U R                  X%;   S[        U5      -   S	-   5        Ub  UR                  5        H  u  p*U
S
:w  ah  U R                  X%;   S[        U5      -   S-   5        U R                  XR   U
:H  S[        U5      -   S-   [        U
5      -   S-   [        XR   5      -   5        Ms  U R                  X%;  S[        U5      -   S-   5        M     Ubj  S
nU H#  n	U[        U5      :X  a    gXU   :X  d  M  US-  nM%     U R                  U[        U5      :H  SU R                  USS9-   S-   [        U5      -   5        gg)zCheck if GraphModule contains the target node
Args:
    graph_module: the GraphModule instance we want to check
    expected_node, expected_node_occurrence, expected_node_list:
       see docs for checkGraphModeFxOp
Fremove_duplicateNr@   rF   rJ   rt   znode:z not found in the graph moduler   zCheck failed for node:z
 not foundz Expected occurrence:z Found occurrence:z! expected no occurrence but foundzCheck failed for graph:)	print_strzExpected ordered list:)dictnamed_modulesrP  nodesr8   r4   r9   r1  r   r  r  itemsr   printGraphModule)r:   graph_moduleexpected_nodeexpected_node_occurrenceexpected_node_listnodes_in_graph	node_listmodulesnoder   
occurrence	cur_indexs               r;   checkGraphModuleNodes*QuantizationTestCase.checkGraphModuleNodes  sR    	|1151IJ &&,,DAww/)TWW-ETWWdkk2M)TWWd7;;+?&@A}  #&"%*%()N% - $OO/#m,,/OO
 $/-E-K-K-M)?OO%703}3EET OO&5C0m,-12 j/* /	/
 n;<= OO%;0m,-=>! .N. )I$6 77955NI	 
 OOS!344)'''FG*+ ()* *r>   c                    [        UR                  SS95      n/ nUR                  R                   H  nSR	                  [        [        UR                  UR                  UR                  UR                  UR                  /5      5      nUR                  S:X  a&  US[        [        X5R                     5      5      -   -  nUR                  U5        M     SR	                  U5      nU(       a  [        U5        U$ )NFr_  rY   rJ   z module type: 
)rb  rc  rP  rd  joinmaprZ   r8   r   r9   r)  r*  r1  r   r   )r:   rg  ra  rm  
node_infosr   	node_infostr_to_prints           r;   rf  %QuantizationTestCase.printGraphModule  s    |1151IJ
##))ATADD!&&!((AFFAHH+U!VWItt}$-T'((:K5L0MMM	i(	 *
 yy,,r>   matched_subgraph_pairsexpected_typesgm_agm_breturnc                 D   S[         S[        S[        [        [        4   4S jnU R                  [        U5      [        U5      :H  S[        U5       S[        U5       35        UR                  5        H  u  pgUu  pUu  pU	u  pX   u  pU" UR                  U5      nU" UR                  U5      nU" UR                  U5      nU" UR                  U5      nU
UL =(       a    UUL =(       a    UUL =(       a    UUL nU R                  USU SXX4 S	UUUU4 35        M     g
)a{  
Verifies that the types specified in expected_types match
the underlying objects pointed to by the nodes in matched_subgraph_pairs.

An example successful test case:

  matched_subgraph_pairs = {'x0': (graph_a_conv_0_node, graph_b_conv_0_node)}
  expected_types = {'x0': (nn.Conv2d, nnq.Conv2d)}

The function tests for key equivalence, and verifies types with
instance checks.
rn  gmr  c                     U R                   S:X  a   [        XR                  5      n[        U5      $ U R                   S;   d   eU R                  $ )NrJ   )r@   rF   )r8   getattrr9   r1  )rn  r  r  s      r;   _get_underlying_op_type]QuantizationTestCase.assert_types_for_matched_subgraph_pairs.<locals>._get_underlying_op_type   sE     77m+!"kk2C9$77&FFFF;;&r>   z-Expected length of results to match, but got  and zType mismatch at z: expected z, got N)
r-   r,   r0   r1   r  r  r   re  
start_nodeend_node)r:   r{  r|  r}  r~  r  r   vexpected_types_aexpected_types_bexp_type_start_aexp_type_end_aexp_type_start_bexp_type_end_b
subgraph_a
subgraph_bact_type_start_aact_type_start_bact_type_end_aact_type_end_btypes_matchs                        r;   'assert_types_for_matched_subgraph_pairs<QuantizationTestCase.assert_types_for_matched_subgraph_pairs
  sm   ,'' +'x}%' OO*+s>/BB?DZ@[?\\abeftbuavw ',,.562 3C0 3C0 )?)B&
#::;P;PRV#W #::;P;PRV#W !89L9Ld!S!89L9Ld!S%)99 ;'>9;)-==; (>9	  's+7GYi6z5{ |,n>NP^_`b! /r>   act_compare_dictc                    UR                  5        GHr  u  p#UR                  5        GHW  u  pEU R                  [        U5      S:H  SU S35        UR                  5       u  pg[	        [        XV   5      5       GH  nXV   U   n	XW   U   n
U R                  U	S   U	S   :H  SU SU SU S35        U R                  [        U	S   5      [        U
S   5      :H  SU SU SU S	35        U[
        R                  R                  :H  =(       a    S
U	S   ;   =(       d    S
U
S   ;   nU(       Gd  [	        [        U	S   5      5       GH  nU	S   U   nU
S   U   n[        U[        R                  5      (       a<  U R                  UR                  UR                  :H  SU SU SU S3SU S3-   5        Mo  [        U[        5      (       aF  US   nUS   nU R                  UR                  UR                  :H  SU SU SU S3SU S3-   5        M  [        U[        5      (       d   S[        U5       35       e[        U5      S:X  d   e[        US   5      S:X  d   eUS   R                  US   R                  :X  d   eUS   S   R                  US   S   R                  :X  d   eUS   S   R                  US   S   R                  :X  a  GM   e   U	S   nU
S   nU	S   nU
S   nU	S   [
        R                  R                  :X  a+  U R                  UU:H  5        U R                  UU:H  5        GM  U	S   [
        R                   R                  :X  d  GM  U R                  UU:g  5        U R                  UU:g  5        GM     GMZ     GMu     g)z
Verifies that the act_compare_dict (output of Numeric Suite APIs) is valid:
1. for each layer, results are recorded for two models
2. number of seen tensors match
3. shapes of each pair of seen tensors match
rp  zLayer z) does not have exactly two model results.r1  z, r  z do not have the same type.r  z- do not have the same number of seen Tensors.conv1dprev_node_target_typerY   zhave a shape mismatch at idx r   r   zunhandled type rt   ref_node_nameprev_node_nameN)re  r  r   r  rx   r&   WEIGHTvaluerS   rh   Tensorrh  listr  r1  NODE_OUTPUT
NODE_INPUT)r:   r  
layer_nameresult_type_to_dataresult_type
layer_datamodel_name_0model_name_1res_idxlayer_data_0layer_data_1is_weight_functional_conv1didxvalues_0values_1ref_node_name_0ref_node_name_1prev_node_name_0prev_node_name_1s                      r;   assert_ns_compare_dict_valid1QuantizationTestCase.assert_ns_compare_dict_validD  s    4D3I3I3K/
/B/H/H/J+KOOJ1, ,UV 2<1B.L#(Z-E)F#G'1'?'H'1'?'H(0L4HH$ZL<.l^Sno
 X 673|H?U;VV$ZL<.l^  TA  B (+C+J+J+P+PP  (L9P,Q Q !U#+|<S/T#T	 4  ;',Sh1G-H'I+7+A#+F+7+A#+F#-h#E#E$(OO(0(..(H*0B|nER^Q__`(a,I#a*P)Q%&
 &0$%?%?/7{H/7{H$(OO(0(..(H*0B|nER^Q__`(a,I#a*P)Q%& ,6(0%,& ,& %J)8h8H'I%J ,& ,/x=A+=$=+=+.x{+;q+@$@+@+3A;+<+<@Q@Q+Q$Q+Q+3A;q>+?+?8A;q>CWCW+W$W+W+3A;q>+?+?8A;q>CWCW+W$W+W3 (J8 +7*G*6*G+78H+I(+78H+I((07CCIIJ !OOO?O,OP OOO?O,OP(07BBHHI !OOO?O,OP OOO?O,OPE $H 0K 4Lr>   c                 x   [        U5      [        L a  US   nU[        R                  :X  a>  [	        [
        R                  R                  R                  5      nUR                  5         OU[        R                  :X  a>  [        [
        R                  R                  R                  5      nUR                  5         O/[        n[        5       R                  U5      nUR                  5         U[        R                  :X  a  [         nO["        nU	bN  [        U	5      [        [$        4;   d   S5       e['        U	[        5      (       a  U	nO[        R(                  " U	5      nU" UUUUUS9nU[        R*                  :w  a  U" U6   U(       a8  [-        5         [-        SU5        [-        SU5        [-        5         [-        SU5        U R/                  UU
UU5        [0        R2                  " U5      n[5        [0        R2                  " U5      5      n[7        [0        R2                  " U5      5      nU" U6 nU" U6 n[0        R2                  " U5      n[0        R2                  " U5      nU(       a  UOUnU(       a1  [-        5         [-        SU5        U R9                  U5        [-        5         U R/                  UUUU5        UUUUUS.$ )	a  Quantizes model with graph mode quantization on fx and check if the
quantized model contains the quantized_node

Args:
    model: floating point torch.nn.Module
    inputs: one positional sample input arguments for model
    expected_node: NodeSpec
        e.g. NodeSpec.call_function(torch.quantize_per_tensor)
    expected_node_occurrence: a dict from NodeSpec to
        expected number of occurrences (int)
        e.g. {NodeSpec.call_function(torch.quantize_per_tensor) : 1,
                NodeSpec.call_method('dequantize'): 1}
    expected_node_list: a list of NodeSpec, used to check the order
        of the occurrence of Node
        e.g. [NodeSpec.call_function(torch.quantize_per_tensor),
                NodeSpec.call_module(nnq.Conv2d),
                NodeSpec.call_function(F.hardtanh_),
                NodeSpec.call_method('dequantize')]
    is_reference: if True, enables reference mode
    print_debug_info: if True, prints debug info
    custom_qconfig_dict: overrides default qconfig_dict
    prepare_expected_node: same as expected_node, but for prepare
    prepare_expected_node_occurrence: same as
        expected_node_occurrence, but for prepare
    prepare_expected_node_list: same as expected_node_list, but
        for prepare

Returns:
    A dictionary with the following structure:
   {
       "prepared": ...,  # the prepared model
       "quantized": ...,  # the quantized non-reference model
       "quantized_reference": ...,  # the quantized reference model
       "result": ...,  # the result for either quantized or
                       # quantized_reference model depending on the
                       # is_reference argument
   }
r   z8custom_qconfig_dict should be a QConfigMapping or a dict)example_inputsprepare_custom_configbackend_configzquant type:
zoriginal model:
zprepared model:
zquantized model:
)r   r$  quantized_referencequantized_outputquantized_reference_output)r1  r  r   r  r   rh   r#  r$  rN  ry   r  r   rO  r   r   
set_globalr+   r*   rb  rS   	from_dictr  r   rq  rQ  rR  r(   r)   rf  )r:   rl   rU  
quant_typerh  ri  rj  is_referenceprint_debug_infocustom_qconfig_dictprepare_expected_node prepare_expected_node_occurrenceprepare_expected_node_listr  r  qconfig_mappingr  preparer   prepared_copyqgraphqgraph_referenceresultresult_referenceqgraph_copyqgraph_reference_copyqgraph_to_checks                              r;   checkGraphModeFxOp'QuantizationTestCase.checkGraphModeFxOp  s_   p F|t#Y]]*"ANN,,33# y///"=NN,,33# 

1"0"2"="=g"F

Y]]*($ #./0"5  N NN  1>BB&9O&4&>&>?R&SO%&;-H Y...&!oz2)51)84&&%0*	 !MM(3Mh 78F6t}}X7NOV_F/8--/K$(MM2B$C!2>.FO*O<%%o6&&("	 *('<$*.> r>   c	                    U(       a  XE/n	OU/n	UR                  5       n
[        R                  " 5       n[        R                  " X5        UR                  S5        [        R                  " U5      n[        R                  R                  R                  nU
 H  n[        X   [        R                  R                  5      (       d  M0  [        X   [        R                  R                  5      (       d   eU" X   5      nU" X   5      nU R                  UU5        M     U(       a  [        R                  " UUSSUS9nO[        R                   " X#US9nU R#                  UUU	5        UR%                  U5        U R                  U" UR&                  R(                  5      U" UR&                  R(                  5      5        U R+                  X/SS9  U(       a"  [        R,                  R                  UUSSSS9nO[        R,                  R!                  X#S	9nU(       a7  [.        R0                  " U[        R2                  SS
9n[5        [6        US9Ul        [;        U5        U" U	6   U(       a"  [        R                  R=                  U5      nSnO![        R                   R=                  U5      nSnU" U	6   U R?                  U[A        U5      ;   5        g )Nr   Tr   )num_embeddingsembedding_diminclude_last_offsetmoder   )r  r  r   )r<  Fr  r  r  scale_grad_by_freqr  r  r  )r   qschemech_axis
activationr"  QuantizedEmbeddingBagQuantizedEmbedding)!r  r  r  rh   r  r  r  rJ  r$  embedding_bag_unpackrS   _CScriptObjectr  r  EmbeddingBag	Embeddingr  r  r  _packed_weightr?  r   r   	with_args per_channel_affine_float_qparamsr   r   r  r   
from_floatr  r  )r:   qembr  r  indicesoffsetsset_qconfig
is_emb_bagr   rU  emb_dictr  r  embedding_unpackkey
emb_weightloaded_weightloaded_qembfloat_embeddingfloat_qparams_observerq_embeddingbagexpected_names                         r;   checkEmbeddingSerialization0QuantizationTestCase.checkEmbeddingSerialization&  sg    'FYF??$JJL

8	q	jjm 99..CCC(-)>)>??!+"2EHH4I4IJJJJ-hm<
 01A B  ];  **-+$(K ---RWK 	&&t[&A##K0T00??@[77FFG	
 	T8TB #hh33-+$(#( 4 O $hh00- 1 O %=%G%GU%K%KUV&" '.9BX'O# 	(  --88IN3M ]]55oFN0M^)<<=r>   )r  rm   r  r  r  r  r  r  r  r  r   NN)F)FFTTFN)NNNT)NNNFFNNNNNN))r]   r^   r_   r`   r  r  r  r  r  r  r  r  r  r  r  r$  r1  r4  r7  r?  r:  rB  r\  rq  rf  HAS_FXrb  r  r  r'   r1   r,   r  r.   r  r  rh   r   r  rc   __classcell__r  s   @r;   r  r  v  s   0Td5'4 OS3j<$0::):95MV: 8z !%IV 8	$(eJ
4J.K)K$L8	 !U58!34eHh<N6OOPP8	 8	 8	 8	tS	Q"3S$sCx.-@(A#ABS	Q S	Qt %)#" $"&-1'+"&K	l llX> X>r>   r  c                       \ rS rSrS\\R                  R                     4S jrS\R                  R                  S\R                  4S jr
Srg)	QuantizationLiteTestCasei  model_classc                     Sn[        U5         [        R                  R                  R	                  U5      nU" S0 UD6n[        U[        U R                  /5      nS S S 5        U$ ! , (       d  f       W$ = f)Nr:  rd   )r#   rh   r  r  r   r   ro   rm   )r:   r  r*  qenginer  rl   s         r;   _create_quantized_model0QuantizationLiteTestCase._create_quantized_model  sh    &w/hh++??HG)&)EU$57HIE	 0  0/ s   AA&&
A5rl   inputc                    Sn[        U5         [        R                  R                  U5      nU" U5      nSn[	        SUS-   5       H  n [
        R                  " UR                  5       5      nUR                  S5        [        U5      n	U	" U5      n
[        R                  R                  XZ5        U	R                  U5      n[        R                  R                  X[5        U	R                  SU5      n[        R                  R                  X\5          O   S S S 5        g ! [         a  nXv:X  a  Ue S nAM  S nAff = f! , (       d  f       g = f)Nr:  r   rt   r   forward)r#   rh   r`  rb  rx   r  r  $_save_to_buffer_for_lite_interpreterr  r"   testingassert_closer
  
run_methodAssertionError)r:   rl   r  r  script_modulescript_module_result	max_retryretryrD  mobile_modulemobile_module_resultmobile_module_forward_resultmobile_module_run_method_resultes                 r;   _compare_script_and_mobile3QuantizationLiteTestCase._compare_script_and_mobile  s-   &w/!II,,U3M#0#7 Iq)a-0!ZZ%JJLF KKN$>v$FM+8+?(MM.., 4A3H3H3O0MM.., 7D6N6N!573 MM.., = 1 0/< & !) 	!= 0/s6   <D?CD D? 
D<*D71D?7D<<D??
Erd   N)r]   r^   r_   r`   r1  rh   r   Moduler  r  r  rc   rd   r>   r;   r  r    s:    	43H 	& & &r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	SingleLayerLinearModeli  r  c                    > [         TU ]  5         [        R                  R	                  SS5      R                  [        R                  S9U l        g Nr   r   )r  r<   rh   r   r  r   r   fc1r:   r  s    r;   r<   SingleLayerLinearModel.__init__  s6    88??1a(++%+++>r>   c                 (    U R                  U5      nU$ rM   r   r:   r  s     r;   r
  SingleLayerLinearModel.forward      HHQKr>   .c                 2    [         R                  " SS5      4$ Nrt   r   rh   r  rO   s    r;   get_example_inputs)SingleLayerLinearModel.get_example_inputs      

1a ""r>   r$  r  Nr]   r^   r_   r`   r<   r
  r  r.   r+  rc   r  r   s   @r;   r  r    s&    ?#E#s(O # #r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	AnnotatedSingleLayerLinearModeli  c                   > [         TU ]  5         [        R                  R                  R                  U5      U l        [        [        R                  R                  SS5      R                  [        R                  S95      U l        g r  )r  r<   rh   r  r  r   r  r   r   r  r   r   r   r:   r  r  s     r;   r<   (AnnotatedSingleLayerLinearModel.__init__  sW    xx,,@@I1 5 8 8u{{ 8 KLr>   c                 (    U R                  U5      nU$ rM   r$  r%  s     r;   r
  'AnnotatedSingleLayerLinearModel.forward  r'  r>   r  .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  2AnnotatedSingleLayerLinearModel.get_example_inputs  r-  r>   r   r  r   r/  r   s   @r;   r1  r1    s'    M
#E#s(O # #r>   r1  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	SingleLayerLinearDynamicModeli  c                    > [         TU ]  5         [        R                  R                  R                  U5      U l        [        R                  R                  SS5      R                  [        R                  S9U l        g r  )r  r<   rh   r  r  r   r  r   r  r   r   r   r3  s     r;   r<   &SingleLayerLinearDynamicModel.__init__  sR    xx,,@@I88??1a(++%+++>r>   c                 (    U R                  U5      nU$ rM   r$  r%  s     r;   r
  %SingleLayerLinearDynamicModel.forward  r'  r>   r  .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  0SingleLayerLinearDynamicModel.get_example_inputs  r-  r>   r9  r:  r/  r   s   @r;   r<  r<    s&    ?
#E#s(O # #r>   r<  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	LinearAddModeli  r  c                 "  > [         TU ]  5         [        R                  R	                  SS5      R                  [        R                  S9U l        [        R                  R	                  SS5      R                  [        R                  S9U l        g Nr      r   	r  r<   rh   r   r  r   r   r   fc2r!  s    r;   r<   LinearAddModel.__init__  ^    88??1a(++%+++>88??1a(++%+++>r>   c                 x    U R                  U5      n[        R                  " US5      nU R                  U5      nU$ Nr   )r   rh   r  rI  r%  s     r;   r
  LinearAddModel.forward  s0    HHQKIIaOHHQKr>   .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  !LinearAddModel.get_example_inputs  r-  r>   r   rI  r.  r/  r   s   @r;   rD  rD    s&    ?
#E#s(O # #r>   rD  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )RNNDynamicModeli  c                 R  > [         TU ]  5         [        U l        US:X  a@  [        R
                  R                  SS5      R                  [        R                  S9U l	        US:X  aA  [        R
                  R                  SS5      R                  [        R                  S9U l	        g g )NGRUrp  r   LSTM)r  r<   r   r  rh   r   rU  r   r   r  rV  r:   mod_typer  s     r;   r<   RNNDynamicModel.__init__  sz    .uxx||Aq),,5;;,?DHvxx}}Q*--EKK-@DH r>   c                 (    U R                  U5      nU$ rM   r  r%  s     r;   r
  RNNDynamicModel.forward  r'  r>   r  r  r]   r^   r_   r`   r<   r
  rc   r  r   s   @r;   rS  rS    s    A r>   rS  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )RNNCellDynamicModeli   c                 f  > [         TU ]  5         [        U l        US:X  a@  [        R
                  R                  SS5      R                  [        R                  S9U l	        US:X  a@  [        R
                  R                  SS5      R                  [        R                  S9U l	        US:X  a?  [        R
                  R                  SSSS9R                  [        R                  S9U l	        US:X  a@  [        R
                  R                  SSS	S9R                  [        R                  S9U l	        g g )
NGRUCellrp  r   LSTMCellRNNReLUrelu)nonlinearityRNNTanhtanh)r  r<   r   r  rh   r   rb  r   r   r  rc  RNNCellrW  s     r;   r<   RNNCellDynamicModel.__init__  s    .y xx''1-00u{{0CDHz!xx((A.111DDHy xx''16'BEEEKKEXDHy xx''16'BEEEKKEXDH !r>   c                 (    U R                  U5      nU$ rM   r[  r%  s     r;   r
  RNNCellDynamicModel.forward  r'  r>   r]  r^  r   s   @r;   r`  r`     s    
Y r>   r`  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )LSTMwithHiddenDynamicModeli  c                    > [         TU ]  5         [        R                  R                  R                  U5      U l        [        R                  R                  SS5      R                  [        R                  S9U l        g )Nrp  r   )r  r<   rh   r  r  r   r  r   rV  r   r   lstmr3  s     r;   r<   #LSTMwithHiddenDynamicModel.__init__  sR    xx,,@@IHHMM!Q'***=	r>   c                 .    U R                  X5      u  pX4$ rM   )rp  )r:   r  hids      r;   r
  "LSTMwithHiddenDynamicModel.forward  s    1"vr>   )rp  r  r:  r^  r   s   @r;   rn  rn    s    >
 r>   rn  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )		ConvModeli  r  c                    > [         TU ]  5         [        R                  R	                  SSSSS9R                  [        R                  S9U l        g Nr  r   Fr#  r   )r  r<   rh   r   Conv2dr   r   convr!  s    r;   r<   ConvModel.__init__  s<    HHOOAq!%O8;;%++;N	r>   c                 (    U R                  U5      nU$ rM   r{  r%  s     r;   r
  ConvModel.forward"      IIaLr>   .c                 6    [         R                  " SSSS5      4$ Nrt   r  r   r*  rO   s    r;   r+  ConvModel.get_example_inputs&      

1aA&((r>   r~  r.  r/  r   s   @r;   rv  rv    s'    O)E#s(O ) )r>   rv  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	ConvTransposeModeli*  r  c                    > [         TU ]  5         [        R                  R	                  SSSSS9R                  [        R                  S9U l        g rx  )r  r<   rh   r   ConvTranspose2dr   r   r{  r!  s    r;   r<   ConvTransposeModel.__init__+  s?    HH,,Q15,ADD5;;DW	r>   c                 (    U R                  U5      nU$ rM   r~  r%  s     r;   r
  ConvTransposeModel.forward/  r  r>   .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  %ConvTransposeModel.get_example_inputs3  r  r>   r~  r.  r/  r   s   @r;   r  r  *  s'    X)E#s(O ) )r>   r  c                   F   ^  \ rS rSrU 4S jrS rS\\S4   4S jrSr	U =r
$ )AnnotatedConvModeli7  c                 :  > [         TU ]  5         [        R                  R                  R                  U5      U l        [        R                  R                  SSSSS9R                  [        R                  S9U l        [        5       U l        [        5       U l        g rx  )r  r<   rh   r  r  r   r  r   rz  r   r   r{  r   r  r   r  r3  s     r;   r<   AnnotatedConvModel.__init__8  sj    xx,,@@IHHOOAq!%O8;;%++;N	[
"}r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r  r{  r  r%  s     r;   r
  AnnotatedConvModel.forward?  .    JJqMIIaLLLOr>   r  .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  %AnnotatedConvModel.get_example_inputsE  r  r>   r{  r  r  r  r/  r   s   @r;   r  r  7  &    %)E#s(O ) )r>   r  c                   F   ^  \ rS rSrU 4S jrS rS\\S4   4S jrSr	U =r
$ )AnnotatedConvTransposeModeliI  c                 :  > [         TU ]  5         [        R                  R                  R                  U5      U l        [        R                  R                  SSSSS9R                  [        R                  S9U l        [        5       U l        [        5       U l        g rx  )r  r<   rh   r  r  r   r  r   r  r   r   r{  r   r  r   r  r3  s     r;   r<   $AnnotatedConvTransposeModel.__init__J  sm    xx,,@@IHH,,Q15,ADD5;;DW	[
"}r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r  r%  s     r;   r
  #AnnotatedConvTransposeModel.forwardQ  r  r>   r  .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  .AnnotatedConvTransposeModel.get_example_inputsW  r  r>   r  r/  r   s   @r;   r  r  I  r  r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	ConvBnModeli[  r  c                    > [         TU ]  5         [        R                  R	                  SSSSS9R                  [        R                  S9U l        [        R                  R                  S5      R                  [        R                  S9U l	        g rx  )
r  r<   rh   r   rz  r   r   r{  BatchNorm2dbnr!  s    r;   r<   ConvBnModel.__init__\  sd    HHOOAq!%O8;;%++;N	((&&q),,5;;,?r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   r{  r  r%  s     r;   r
  ConvBnModel.forwarda  !    IIaLGGAJr>   .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  ConvBnModel.get_example_inputsf  r  r>   r  r{  r.  r/  r   s   @r;   r  r  [  s'    @

)E#s(O ) )r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	AnnotatedConvBnModelij  r  c                 r  > [         TU ]  5         [        U l        [        R
                  R                  SSSSS9R                  [        R                  S9U l	        [        R
                  R                  S5      R                  [        R                  S9U l        [        5       U l        [        5       U l        g rx  )r  r<   r   r  rh   r   rz  r   r   r{  r  r  r   r  r   r  r!  s    r;   r<   AnnotatedConvBnModel.__init__k  s}    &HHOOAq!%O8;;%++;N	((&&q),,5;;,?[
"}r>   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ rM   )r  r{  r  r  r%  s     r;   r
  AnnotatedConvBnModel.forwards  s;    JJqMIIaLGGAJLLOr>   .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  'AnnotatedConvBnModel.get_example_inputsz  r  r>   )r  r{  r  r  r  r.  r/  r   s   @r;   r  r  j  s&    %)E#s(O ) )r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	ConvBnReLUModeli~  r  c                 R  > [         TU ]  5         [        R                  R	                  SSSSS9R                  [        R                  S9U l        [        R                  R                  S5      R                  [        R                  S9U l	        [        R                  " SS9U l        g Nr  r   Fry  r   Tr   )r  r<   rh   r   rz  r   r   r{  r  r  ReLUre  r!  s    r;   r<   ConvBnReLUModel.__init__  st    HHOOAq!%O8;;%++;N	((&&q),,5;;,?GGD)	r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r{  r  re  r%  s     r;   r
  ConvBnReLUModel.forward  s.    IIaLGGAJIIaLr>   .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  "ConvBnReLUModel.get_example_inputs  r  r>   r  r{  re  r.  r/  r   s   @r;   r  r  ~  s&    *)E#s(O ) )r>   r  c                   P   ^  \ rS rSrS	U 4S jjrS rS rS\\S4   4S jr	Sr
U =r$ )
AnnotatedConvBnReLUModeli  c                   > [         TU ]  5         [        R                  R                  R                  U5      U l        [        R                  R                  SSSSS9R                  [        R                  S9U l        [        R                  R                  S5      R                  [        R                  S9U l        [        R                  " SS9U l        [!        5       U l        [%        5       U l        g r  )r  r<   rh   r  r  r   r  r   rz  r   r   r{  r  r  r  re  r   r  r   r  r3  s     r;   r<   !AnnotatedConvBnReLUModel.__init__  s    xx,,@@IHHOOAq!%O8;;%++;N	((&&q),,5;;,?GGD)	[
"}r>   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU$ rM   )r  r{  r  re  r  r%  s     r;   r
   AnnotatedConvBnReLUModel.forward  sH    JJqMIIaLGGAJIIaLLLOr>   c                     U R                   (       a-  [        R                  R                  R	                  U / SQ/SS9  g [        R                  R                  R                  U / SQ/SS9  g )Nr  Tr   trainingrh   r  r  fuse_modules_qatfuse_modulesrO   s    r;   
fuse_model#AnnotatedConvBnReLUModel.fuse_model  s[    ==HH!!22-. 3  HH!!..-. / r>   r  .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  +AnnotatedConvBnReLUModel.get_example_inputs  r  r>   )r  r{  r  r  r  re  r:  )r]   r^   r_   r`   r<   r
  r  r  r.   r+  rc   r  r   s   @r;   r  r    s+    %	)E#s(O ) )r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	TwoLayerConvModeli  r  c                 "  > [         TU ]  5         [        R                  R	                  SSSSS9R                  [        R                  S9U l        [        R                  R	                  SSSSS9R                  [        R                  S9U l        g )Nr  r   Fry  r   rt   )	r  r<   rh   r   rz  r   r   conv1conv2r!  s    r;   r<   TwoLayerConvModel.__init__  sj    XX__Q15_9<<5;;<O
XX__Q15_9<<5;;<O
r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   r  r  r%  s     r;   r
  TwoLayerConvModel.forward  !    JJqMJJqMr>   .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  $TwoLayerConvModel.get_example_inputs  r  r>   r  r.  r/  r   s   @r;   r  r    s'    P

)E#s(O ) )r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	TwoLayerLinearModeli  r  c                 "  > [         TU ]  5         [        R                  R	                  SS5      R                  [        R                  S9U l        [        R                  R	                  SS5      R                  [        R                  S9U l        g rF  rH  r!  s    r;   r<   TwoLayerLinearModel.__init__  rK  r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   rQ  r%  s     r;   r
  TwoLayerLinearModel.forward  !    HHQKHHQKr>   .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  &TwoLayerLinearModel.get_example_inputs  r-  r>   rQ  r.  r/  r   s   @r;   r  r    s&    ?

#E#s(O # #r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	LinearModelWithSubmodulei  r  c                 x   > [         TU ]  5         [        5       U l        [        R
                  " SS5      U l        g rM  )r  r<   r  submr   r  fcr!  s    r;   r<   !LinearModelWithSubmodule.__init__  s)    ')	))Aq/r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   )r  r  r%  s     r;   r
   LinearModelWithSubmodule.forward  r  r>   .c                 6    U R                   R                  5       $ rM   )r  r+  rO   s    r;   r+  +LinearModelWithSubmodule.get_example_inputs      yy++--r>   )r  r  r.  r/  r   s   @r;   r  r    s&    "

.E#s(O . .r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	AnnotatedTwoLayerLinearModeli  r  c                   > [         TU ]  5         [        R                  R	                  SS5      R                  [        R                  S9U l        [        [        R                  R	                  SS5      R                  [        R                  S95      U l	        [        R                  R                  R                  S5      U R                  l        g )Nr   rG  r   r   )r  r<   rh   r   r  r   r   r   r   rI  r  r  r   r  r!  s    r;   r<   %AnnotatedTwoLayerLinearModel.__init__  s    88??1a(++%+++>1 5 8 8u{{ 8 KL 8800DDXNr>   c                 J    U R                  U5      nU R                  U5      nU$ rM   rQ  r%  s     r;   r
  $AnnotatedTwoLayerLinearModel.forward  r  r>   .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  /AnnotatedTwoLayerLinearModel.get_example_inputs  r-  r>   rQ  r.  r/  r   s   @r;   r  r    s'    O
#E#s(O # #r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )ActivationsTestModeli  c                 *  > [         TU ]  5         [        R                  R                  R                  S5      U l        [        R                  R                  R                  5       U l        [        R                  R                  5       R                  [        R                  S9U l        [        R                  R                  5       R                  [        R                  S9U l        [        R                  R                  R!                  5       U l        g )Nr   r   )r  r<   rh   r  r  r   r  r   r  r   	Hardswishr   r   	hardswishELUelur   r  r!  s    r;   r<   ActivationsTestModel.__init__  s    xx,,@@JXX**446
++-00u{{0C88<<>$$5;;$7xx,,88:r>   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ rM   )r  r  r   r  r%  s     r;   r
  ActivationsTestModel.forward  s<    JJqMNN1HHQKLLOr>   )r  r   r  r  r  r.  r^  r   s   @r;   r  r    s    ; r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	LinearReluModeli  r  c                    > [         TU ]  5         [        R                  R	                  SS5      R                  [        R                  S9U l        [        R                  R                  5       U l	        g r  
r  r<   rh   r   r  r   r   r  r  re  r!  s    r;   r<   LinearReluModel.__init__  sG    ((//!Q'***=HHMMO	r>   c                 F    U R                  U R                  U5      5      nU$ rM   re  r  r%  s     r;   r
  LinearReluModel.forward      IIdggaj!r>   .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  "LinearReluModel.get_example_inputs  r-  r>   r  re  r.  r/  r   s   @r;   r  r    s&    $
#E#s(O # #r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	LinearReluLinearModeli  r  c                 h  > [         TU ]  5         [        R                  R	                  SS5      R                  [        R                  S9U l        [        R                  R                  5       U l	        [        R                  R	                  SS5      R                  [        R                  S9U l
        g rF  r  r<   rh   r   r  r   r   r   r  re  rI  r!  s    r;   r<   LinearReluLinearModel.__init__  o    88??1a(++%+++>HHMMO	88??1a(++%+++>r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r   re  rI  r%  s     r;   r
  LinearReluLinearModel.forward  .    HHQKIIaLHHQKr>   .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  (LinearReluLinearModel.get_example_inputs  r-  r>   r   rI  re  r.  r/  r   s   @r;   r  r    s&    ?#E#s(O # #r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	LinearReluAddModeli  r  c                 h  > [         TU ]  5         [        R                  R	                  SS5      R                  [        R                  S9U l        [        R                  R                  5       U l	        [        R                  R	                  SS5      R                  [        R                  S9U l
        g r  r  r!  s    r;   r<   LinearReluAddModel.__init__  r  r>   c                     U R                  U5      nU R                  U5      n[        R                  " US5      nU R	                  U5      n[        R
                  R                  5       U l        U$ rM  r   re  rh   r  rI  r   r  r%  s     r;   r
  LinearReluAddModel.forward$  N    HHQKIIaLIIaOHHQKHHMMO	r>   .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  %LinearReluAddModel.get_example_inputs,  r-  r>   r  r.  r/  r   s   @r;   r  r    s&    ?#E#s(O # #r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	LinearBnLeakyReluModeli0  c                    > [         TU ]  5         [        R                  " SS5      U l        [        R
                  " S5      U l        [        R                  " S5      U l        Xl	        g )Nr   g{Gz?)
r  r<   r   r  linearBatchNorm1dbn1d	LeakyReLU
leaky_reluwith_bn)r:   r/  r  s     r;   r<   LinearBnLeakyReluModel.__init__1  sD    ii1oNN1%	,,t,r>   c                     U R                  U5      nU R                  (       a  U R                  U5      nU R                  U5      nU$ rM   )r*  r/  r,  r.  r%  s     r;   r
  LinearBnLeakyReluModel.forward8  s6    KKN<<		!AOOAr>   r  .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  )LinearBnLeakyReluModel.get_example_inputs?  r-  r>   )r,  r.  r*  r/  r  r/  r   s   @r;   r(  r(  0  s&    #E#s(O # #r>   r(  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	LinearTanhModeliC  r  c                    > [         TU ]  5         [        R                  " SS5      U l        [        R
                  " 5       U l        g rM  )r  r<   r   r  r*  Tanhrh  r!  s    r;   r<   LinearTanhModel.__init__D  s,    ii1oGGI	r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   r*  rh  r%  s     r;   r
  LinearTanhModel.forwardI  s!    KKNIIaLr>   .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  "LinearTanhModel.get_example_inputsN  r-  r>   r;  r.  r/  r   s   @r;   r6  r6  C  s&    

#E#s(O # #r>   r6  c                   T   ^  \ rS rSr     SU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	ConvBnAddReluModeliR  c                 <  > [         TU ]  5         [        R                  " SSS5      U l        [        R                  " SSS5      U l        [        R                  " S5      U l        [        R                  " 5       U l	        Xl
        X l        X@l        X0l        XPl        g )Nr   )rp  rp  )r  r<   r   rz  r{  r  r  r  r  re  r/  	with_relutwo_conv	left_convuse_torch_add)r:   r/  rB  rD  rC  rE  r  s         r;   r<   ConvBnAddReluModel.__init__S  so     	IIaF+	YYq!V,
..#GGI	" "*r>   c                    U R                   (       Ga	  U R                  (       a  U R                  (       aF  [        R                  " U R                  U R                  U5      5      U R                  U5      5      nGOA[        R                  " U R                  U5      U R                  U5      5      nGO
U R                  (       a4  U R                  U R                  U5      5      U R                  U5      -   nGOU R                  U5      U R                  U5      -   nGOU R                  (       a  U R                  (       ap  U R                  (       a7  [        R                  " U R                  U R                  U5      5      U5      nGO6[        R                  " U R                  U5      U5      nGOU R                  (       a5  [        R                  " X R                  U R                  U5      5      5      nO[        R                  " X R                  U5      5      nOU R                  (       aJ  U R                  (       a$  U R                  U R                  U5      5      U-   nO\U R                  U5      U-   nOGU R                  (       a#  X R                  U R                  U5      5      -   nOX R                  U5      -   nU R                  (       a  U R                  U5      nU$ rM   )rC  rE  r/  rh   r  r  r{  r  rD  rB  re  )r:   x1x2r  s       r;   r
  ConvBnAddReluModel.forwardf  s   ===!!<<		$''$))B-"8$**R.IA		$))B-B@A<<		".B?A		"

26A!!>>||!IIdggdiim&<bA!IIdiimR8||!IIb''$))B-*@A!IIb))B-8>>|| GGDIIbM2R7 IIbMB.||2!772.>>		!Ar>   r  .c                 f    [         R                  " SSSS5      [         R                  " SSSS5      4$ )Nrt   r   r  rp  r*  rO   s    r;   r+  %ConvBnAddReluModel.get_example_inputs  s+    

1aA&

1aA(>??r>   )	r  r{  r  rD  re  rC  rE  r/  rB  )TTTTTr/  r   s   @r;   r@  r@  R  s;     +&%N@E#s(O @ @r>   r@  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	ConvReluModeli  r  c                    > [         TU ]  5         [        R                  R	                  SSS5      R                  [        R                  S9U l        [        R                  R                  5       U l	        g )Nr  r   r   )
r  r<   rh   r   rz  r   r   r  r  re  r!  s    r;   r<   ConvReluModel.__init__  sI    ((//!Q*--EKK-@HHMMO	r>   c                 F    U R                  U R                  U5      5      nU$ rM   r
  r%  s     r;   r
  ConvReluModel.forward  r  r>   .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+   ConvReluModel.get_example_inputs  r  r>   r  r.  r/  r   s   @r;   rN  rN    s&    $
)E#s(O ) )r>   rN  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	ConvReluConvModeli  r  c                 l  > [         TU ]  5         [        R                  R	                  SSS5      R                  [        R                  S9U l        [        R                  R                  5       U l	        [        R                  R	                  SSS5      R                  [        R                  S9U l
        g Nr  r   r   rt   r  r<   rh   r   rz  r   r   r   r  re  rI  r!  s    r;   r<   ConvReluConvModel.__init__  s    88??1a+..U[[.AHHMMO	88??1a+..U[[.Ar>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r  r%  s     r;   r
  ConvReluConvModel.forward  r  r>   .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  $ConvReluConvModel.get_example_inputs  r  r>   r  r.  r/  r   s   @r;   rV  rV    s'    B)E#s(O ) )r>   rV  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	ConvReluAddModeli  r  c                 l  > [         TU ]  5         [        R                  R	                  SSS5      R                  [        R                  S9U l        [        R                  R                  5       U l	        [        R                  R	                  SSS5      R                  [        R                  S9U l
        g rX  rY  r!  s    r;   r<   ConvReluAddModel.__init__  r[  r>   c                     U R                  U5      nU R                  U5      n[        R                  " US5      nU R	                  U5      n[        R
                  R                  5       U l        U$ rM  r"  r%  s     r;   r
  ConvReluAddModel.forward  r$  r>   .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  #ConvReluAddModel.get_example_inputs  r  r>   r  r.  r/  r   s   @r;   ra  ra    s'    B)E#s(O ) )r>   ra  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )NormalizationTestModeli  c                 f  > [         TU ]  5         [        R                  R                  R                  5       U l        [        R                  R                  SS5      R                  [        R                  S9U l        [        R                  R                  S5      U l        [        R                  R                  SS5      U l        [        R                  R!                  S5      U l        [        R                  R%                  S5      U l        [        R                  R)                  S5      U l        g )Nr   rG  r   rp  )r  r<   rh   r  r  r   r  r   r  r   r   r   	LayerNorm
layer_norm	GroupNorm
group_normInstanceNorm1dinstance_norm1dInstanceNorm2dinstance_norm2dInstanceNorm3dinstance_norm3dr!  s    r;   r<   NormalizationTestModel.__init__  s    XX**446
88??1a(++%+++>((,,Q/((,,Q2$xx66q9$xx66q9$xx66q9r>   c                 p   U R                  U5      nU R                  U5      nU R                  U5      nU R                  UR	                  S5      R                  SSS5      5      nU R                  U5      nU R                  UR	                  S5      5      nU R                  UR	                  S5      5      nU$ )Nr   rt   r  )	r  r   rl  rn  r  repeatrp  rr  rt  r%  s     r;   r
  NormalizationTestModel.forward  s    JJqMHHQKOOAOOAKKO221a;<  #  R1  R1r>   )r   rn  rp  rr  rt  rl  r  r.  r^  r   s   @r;   ri  ri    s    : r>   ri  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )NestedModeli  c                    > [         TU ]  5         [        5       U l        [	        5       U l        [        R                  R                  SS5      R                  [        R                  S9U l        g r  )r  r<   r  sub1r  sub2rh   r   r  r   r   fc3r!  s    r;   r<   NestedModel.__init__  sJ    #%	')	88??1a(++%+++>r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r|  r}  r~  r%  s     r;   r
  NestedModel.forward  .    IIaLIIaLHHQKr>   r~  r|  r}  r.  r^  r   s   @r;   rz  rz    s    ? r>   rz  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )AnnotatedNestedModeli  c                    > [         TU ]  5         [        5       U l        [	        5       U l        [        [        R                  R                  SS5      R                  [        R                  S95      U l        [        U R                  l        [        U R
                  R                  5      U R
                  l        US:X  a   [         U R
                  R                  l        g [        U R
                  R                  l        g )Nr   r   r   )r  r<   r  r|  r  r}  r   rh   r   r  r   r   r~  r   r  r   r   r3  s     r;   r<   AnnotatedNestedModel.__init__  s    #%	')	1 5 8 8u{{ 8 KL*$TYY]]3		h$?DIIMM!$3DIIMM!r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r  r%  s     r;   r
  AnnotatedNestedModel.forward  r  r>   r  r^  r   s   @r;   r  r    s    
4 r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )AnnotatedSubNestedModeli  c                 V  > [         TU ]  5         [        5       U l        [	        [        5       5      U l        [	        [        R                  R                  SS5      R                  [        R                  S95      U l        [        U R                  l        [        U R                  l        g r  )r  r<   r  r|  r   r  r}  rh   r   r  r   r   r~  r   r  r!  s    r;   r<    AnnotatedSubNestedModel.__init__  sm    #%	 !4!67	1 5 8 8u{{ 8 KL*+		r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r  r%  s     r;   r
  AnnotatedSubNestedModel.forward  r  r>   r  r.  r^  r   s   @r;   r  r    s    , r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ ) AnnotatedCustomConfigNestedModeli  c                   > [         TU ]  5         [        5       U l        [	        5       U l        [        [        R                  R                  SS5      R                  [        R                  S95      U l        [        U R                  l        [        U R
                  l        [        R                  [        R                   S.n[#        [$        R&                  " S0 UD6[(        S9nX R
                  R*                  l        [        U R
                  R*                  5      U R
                  l        [        U R
                  R,                  5      U R
                  l        g )Nr   r   )r   r  r  rd   )r  r<   r  r|  r  r}  r   rh   r   r  r   r   r~  r   r  r   per_tensor_affiner   r
   r  r   r   rI  )r:   custom_optionscustom_qconfigr  s      r;   r<   )AnnotatedCustomConfigNestedModel.__init__  s    #%	')	1 5 8 8u{{ 8 KL*+		#(<<E<S<ST '11CNC*
 !/		$TYY]]3		$TYY]]3		r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r  r%  s     r;   r
  (AnnotatedCustomConfigNestedModel.forward!  r  r>   r  r.  r^  r   s   @r;   r  r    s    4$ r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )QuantSubModeli(  c                 D  > [         TU ]  5         [        5       U l        [	        [        5       5      U l        [        U R                  l        [        R                  R                  SS5      R                  [        R                  S9U l        [        U R                  l        g r  )r  r<   r  r|  r   r  r}  r   r  rh   r   r  r   r   r~  r!  s    r;   r<   QuantSubModel.__init__)  sh    #%	 !4!67	+		88??1a(++%+++>*r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   r  r%  s     r;   r
  QuantSubModel.forward1  r  r>   r  r.  r^  r   s   @r;   r  r  (  s    + r>   r  c                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )InnerModulei8  c                   > [         TU ]  5         [        R                  R	                  SS5      R                  [        R                  S9U l        [        R                  R                  5       U l	        [        R                  R	                  SS5      R                  [        R                  S9U l
        [        R                  R                  5       U l        g rF  )r  r<   rh   r   r  r   r   r   r  relu1rI  relu2r!  s    r;   r<   InnerModule.__init__9  s    88??1a(++%+++>XX]]_
88??1a(++%+++>XX]]_
r>   c           	      ~    U R                  U R                  U R                  U R                  U5      5      5      5      $ rM   )r  rI  r  r   r%  s     r;   r
  InnerModule.forward@  s+    zz$((4::dhhqk#:;<<r>   c                 F   / n[        U R                  5       5      n[        U5       H  u  nu  pE[        U[        R
                  R                  5      (       d  M3  U[        U5      S-
  :  a    OR[        X#S-      S   [        R
                  R                  5      (       d  Mz  UR                  XBUS-      S   /5        M     U R                  (       a)  [        R                  R                  R                  XSS9  g [        R                  R                  R                  XSS9  g )Nrt   r   Tr   )r  r  r   rS   rh   r   r  r   r  r   r  r  r  r  r  )r:   fusable_layersr  r  current_namelayers         r;   r  InnerModule.fuse_modulesC  s    d1134*3N*C&C&,%11#n-11n1W5a8%((--HH"))<a9PQR9S*TU +D ==HH!!224QU2VHH!!..tT.Rr>   )r   rI  r  r  r.  )	r]   r^   r_   r`   r<   r
  r  rc   r  r   s   @r;   r  r  8  s    %=S Sr>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	FunctionalLineariS  r  c                    > [         TU ]  5         [        R                  " S5      U l        [        R
                  " S5      U l        g )N)r   r   r   )r  r<   rh   r  r"  rg  r#  r!  s    r;   r<   FunctionalLinear.__init__T  s-    jj(KKN	r>   c                 X    [         R                  " XR                  U R                  5      $ rM   )Fr*  r"  r#  r%  s     r;   r
  FunctionalLinear.forwardY  s    xx;;		22r>   .c                 2    [         R                  " SS5      4$ r)  r*  rO   s    r;   r+  #FunctionalLinear.get_example_inputs\  r-  r>   )r#  r"  r.  r/  r   s   @r;   r  r  S  s&    #
3#E#s(O # #r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	 SingleLayerFunctionalLinearModeli`  r  c                 @   > [         TU ]  5         [        5       U l        g rM   )r  r<   r  linear1r!  s    r;   r<   )SingleLayerFunctionalLinearModel.__init__a  s    ')r>   c                 (    U R                  U5      nU$ rM   r  r%  s     r;   r
  (SingleLayerFunctionalLinearModel.forwarde  s    LLOr>   .c                 6    U R                   R                  5       $ rM   r  r+  rO   s    r;   r+  3SingleLayerFunctionalLinearModel.get_example_inputsi      ||..00r>   r  r.  r/  r   s   @r;   r  r  `  s&    *1E#s(O 1 1r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	TwoLayerFunctionalLinearModelim  r  c                 ^   > [         TU ]  5         [        5       U l        [        5       U l        g rM   r  r<   r  r  linear2r!  s    r;   r<   &TwoLayerFunctionalLinearModel.__init__n  "    ')')r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   r  r  r%  s     r;   r
  %TwoLayerFunctionalLinearModel.forwards  s!    LLOLLOr>   .c                 6    U R                   R                  5       $ rM   r  rO   s    r;   r+  0TwoLayerFunctionalLinearModel.get_example_inputsx  r  r>   r  r.  r/  r   s   @r;   r  r  m  s&    *

1E#s(O 1 1r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	FunctionalLinearAddModeli|  r  c                 ^   > [         TU ]  5         [        5       U l        [        5       U l        g rM   r  r!  s    r;   r<   !FunctionalLinearAddModel.__init__}  r  r>   c                 x    U R                  U5      n[        R                  " US5      nU R                  U5      nU$ rM  )r  rh   r  r  r%  s     r;   r
   FunctionalLinearAddModel.forward  s0    LLOIIaOLLOr>   .c                 6    U R                   R                  5       $ rM   r  rO   s    r;   r+  +FunctionalLinearAddModel.get_example_inputs  r  r>   r  r.  r/  r   s   @r;   r  r  |  s&    *
1E#s(O 1 1r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	FunctionalLinearReluModeli  r  c                 @   > [         TU ]  5         [        5       U l        g rM   )r  r<   r  r*  r!  s    r;   r<   "FunctionalLinearReluModel.__init__  s    &(r>   c                 T    U R                  U5      n[        R                  " U5      nU$ rM   )r*  r  re  r%  s     r;   r
  !FunctionalLinearReluModel.forward  s!    KKNFF1Ir>   .c                 6    U R                   R                  5       $ rM   )r*  r+  rO   s    r;   r+  ,FunctionalLinearReluModel.get_example_inputs  s    {{--//r>   )r*  r.  r/  r   s   @r;   r  r    s&    )
0E#s(O 0 0r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	FunctionalLinearReluLinearModeli  r  c                    > [         TU ]  5         [        5       U l        [        R
                  " 5       U l        [        5       U l        g rM   )r  r<   r  r  r   r  re  r  r!  s    r;   r<   (FunctionalLinearReluLinearModel.__init__  s/    ')GGI	')r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   )r  re  r  r%  s     r;   r
  'FunctionalLinearReluLinearModel.forward  s.    LLOIIaLLLOr>   .c                 6    U R                   R                  5       $ rM   r  rO   s    r;   r+  2FunctionalLinearReluLinearModel.get_example_inputs  r  r>   )r  r  re  r.  r/  r   s   @r;   r  r    s&    *1E#s(O 1 1r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	FunctionalConv2di  r  c                    > [         TU ]  5         [        R                  " SSSS5      U l        [        R                  " S5      U l        SU l        SU l        SU l        SU l	        g )Nr  rt   rt   )r   r   rt   )
r  r<   rh   r  r"  r#  stridepaddingdilationr  r!  s    r;   r<   FunctionalConv2d.__init__  sO    jjAq!,JJqM	r>   c           	          [         R                  " UU R                  U R                  U R                  U R
                  U R                  U R                  5      $ rM   )r  conv2dr"  r#  r  r  r  r  r%  s     r;   r
  FunctionalConv2d.forward  s@    xxKKIIKKLLMMKK
 	
r>   .c                 6    [         R                  " SSSS5      4$ r  r*  rO   s    r;   r+  #FunctionalConv2d.get_example_inputs  r  r>   )r#  r  r  r  r  r"  r.  r/  r   s   @r;   r  r    s&    	
)E#s(O ) )r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	SingleLayerFunctionalConvModeli  r  c                 @   > [         TU ]  5         [        5       U l        g rM   )r  r<   r  r  r!  s    r;   r<   'SingleLayerFunctionalConvModel.__init__  s    %'
r>   c                 (    U R                  U5      nU$ rM   r  r%  s     r;   r
  &SingleLayerFunctionalConvModel.forward  s    JJqMr>   .c                 6    U R                   R                  5       $ rM   r  r+  rO   s    r;   r+  1SingleLayerFunctionalConvModel.get_example_inputs      zz,,..r>   r  r.  r/  r   s   @r;   r  r    s&    (/E#s(O / /r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	TwoLayerFunctionalConvModeli  r  c                 ^   > [         TU ]  5         [        5       U l        [        5       U l        g rM   )r  r<   r  r  r  r!  s    r;   r<   $TwoLayerFunctionalConvModel.__init__  s"    %'
%'
r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   r  r%  s     r;   r
  #TwoLayerFunctionalConvModel.forward  r  r>   .c                 6    U R                   R                  5       $ rM   r  rO   s    r;   r+  .TwoLayerFunctionalConvModel.get_example_inputs  r  r>   r  r.  r/  r   s   @r;   r  r    s&    (

/E#s(O / /r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	FunctionalConvReluModeli  r  c                 @   > [         TU ]  5         [        5       U l        g rM   )r  r<   r  r{  r!  s    r;   r<    FunctionalConvReluModel.__init__  s    $&	r>   c                 T    U R                  U5      n[        R                  " U5      nU$ rM   )r{  r  re  r%  s     r;   r
  FunctionalConvReluModel.forward  s!    IIaLFF1Ir>   .c                 6    U R                   R                  5       $ rM   )r{  r+  rO   s    r;   r+  *FunctionalConvReluModel.get_example_inputs  r  r>   r~  r.  r/  r   s   @r;   r  r    s&    '
.E#s(O . .r>   r  c                   J   ^  \ rS rSrSU 4S jjrS rS\\S4   4S jrSr	U =r
$ )	FunctionalConvReluConvModeli  r  c                    > [         TU ]  5         [        5       U l        [        R
                  " 5       U l        [        5       U l        g rM   )r  r<   r  r  r   r  re  r  r!  s    r;   r<   $FunctionalConvReluConvModel.__init__  s/    %'
GGI	%'
r>   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rM   )r  re  r  r%  s     r;   r
  #FunctionalConvReluConvModel.forward  s.    JJqMIIaLJJqMr>   .c                 6    U R                   R                  5       $ rM   r  rO   s    r;   r+  .FunctionalConvReluConvModel.get_example_inputs  r  r>   )r  r  re  r.  r/  r   s   @r;   r  r    s&    (/E#s(O / /r>   r  c                   <   ^  \ rS rSrSrSU 4S jjrS rS rSrU =r	$ )SkipQuantModeli  NWe can skip quantization by explicitly
setting qconfig of a submodule to None
c                    > [         TU ]  5         [        5       U l        [        R
                  R                  SS5      R                  [        R                  S9U l	        g r  )
r  r<   r  r{  rh   r   r  r   r   r  r!  s    r;   r<   SkipQuantModel.__init__	  s?    =((//!Q'***=r>   c                 B    U R                  U R                  U5      5      $ rM   r  r{  r%  s     r;   r
  SkipQuantModel.forward		      wwtxx{##r>   c                 8    U R                   R                  5         g rM   )r{  r  rO   s    r;   r  SkipQuantModel.fuse_modules	  s    r>   r  r.  
r]   r^   r_   r`   ra   r<   r
  r  rc   r  r   s   @r;   r  r    s    >
$   r>   r  c                   8   ^  \ rS rSrSrU 4S jrS rS rSrU =r	$ )AnnotatedSkipQuantModeli	  r  c                 P  > [         TU ]  5         [        R                  R                  R                  U5      U l        [        [        5       5      U l	        [        R                  R                  SS5      R                  [        R                  S9U l        S U R                  l        g r  )r  r<   rh   r  r  r   r  r   r  r{  r   r  r   r   r  r3  s     r;   r<    AnnotatedSkipQuantModel.__init__	  sk    xx,,@@I.((//!Q'***=r>   c                 B    U R                  U R                  U5      5      $ rM   r  r%  s     r;   r
  AnnotatedSkipQuantModel.forward	  r  r>   c                 L    U R                   R                  R                  5         g rM   )r{  r   r  rO   s    r;   r  $AnnotatedSkipQuantModel.fuse_modules 	  s    $$&r>   )r  r  r{  r!  r   s   @r;   r#  r#  	  s    $' 'r>   r#  c                   6   ^  \ rS rSrSrSU 4S jjrS rSrU =r$ )QuantStubModeli$	  =A Module with manually inserted `QuantStub` and `DeQuantStub`c                 :  > [         TU ]  5         [        R                  R                  R                  S5      U l        [        5       U l        [        5       U l
        [        R                  R                  SS5      R                  [        R                  S9U l        g )Nr:  r   r   )r  r<   rh   r  r  r   r  r   r  r   r  r   r  r   r   r  r!  s    r;   r<   QuantStubModel.__init__'	  sd    xx,,@@K[
"}((//!Q'***=r>   c                 h    U R                  U5      nU R                  U5      nU R                  U5      $ rM   )r  r  r  r%  s     r;   r
  QuantStubModel.forward.	  s*    JJqMGGAJ||Ar>   )r  r  r  r  r.  	r]   r^   r_   r`   ra   r<   r
  rc   r  r   s   @r;   r+  r+  $	  s    H> r>   r+  c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )ManualLinearQATModeli4	  r,  c                   > [         TU ]  5         [        R                  R                  R                  U5      U l        [        5       U l        [        5       U l
        [        R                  R                  SS5      R                  [        R                  S9U l        [        R                  R                  SS5      R                  [        R                  S9U l        g Nr   rt   r   rs   )r  r<   rh   r  r  r   r  r   r  r   r  r   r  r   r   r   rI  r3  s     r;   r<   ManualLinearQATModel.__init__7	  s    xx,,DDWM[
"}88??1a(++%+++>88??1b),,5;;,?r>   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      $ rM   )r  r   rI  r  r%  s     r;   r
  ManualLinearQATModel.forward?	  s7    JJqMHHQKHHQK||Ar>   )r  r   rI  r  r  r1  r   s   @r;   r3  r3  4	  s    H@ r>   r3  c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )ManualDropoutQATModeliF	  r,  c                   > [         TU ]  5         [        R                  R                  R                  U5      U l        [        5       U l        [        5       U l
        [        R                  R                  SS5      R                  [        R                  S9U l        [        R                  R!                  S5      U l        g )Nr   rt   r         ?)r  r<   rh   r  r  r   r  r   r  r   r  r   r  r   r   r   r  dropoutr3  s     r;   r<   ManualDropoutQATModel.__init__I	  sz    xx,,DDWM[
"}88??1a(++%+++>xx'',r>   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      $ rM   )r  r   r=  r  r%  s     r;   r
  ManualDropoutQATModel.forwardQ	  s7    JJqMHHQKLLO||Ar>   )r  r=  r   r  r  r1  r   s   @r;   r:  r:  F	  s    H- r>   r:  c                   6   ^  \ rS rSrSrSU 4S jjrS rSrU =r$ )ManualLinearDynamicQATModeliX	  z,A Module that uses a dynamic QAT by default.c                 J  > [         TU ]  5         U=(       d    [        U l        [        R
                  R                  SS5      R                  [        R                  S9U l	        [        R
                  R                  SS5      R                  [        R                  S9U l
        g r5  )r  r<   r   r  rh   r   r  r   r   r   rI  r:   r  r  s     r;   r<   $ManualLinearDynamicQATModel.__init__[	  sk    ="=88??1a(++%+++>88??1b),,5;;,?r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   rQ  r%  s     r;   r
  #ManualLinearDynamicQATModel.forwarda	  r  r>   )r   rI  r  rM   r1  r   s   @r;   rB  rB  X	  s    7@ r>   rB  c                   6   ^  \ rS rSrSrSU 4S jjrS rSrU =r$ )ManualConvLinearQATModelig	  zhA module with manually inserted `QuantStub` and `DeQuantStub`
and contains both linear and conv modules
c                 J  > [         TU ]  5         U(       a  UO([        R                  R                  R                  S5      U l        [        5       U l        [        5       U l
        [        R                  R                  SSSS9R                  [        R                  S9U l        [        R                  R!                  SS5      R                  [        R                  S9U l        [        R                  R!                  SS5      R                  [        R                  S9U l        g )Nr:  r  rt   )r  r   @   rs   )r  r<   rh   r  r  r   r  r   r  r   r  r   rz  r   r   r{  r  r   rI  rD  s     r;   r<   !ManualConvLinearQATModel.__init__l	  s      &&>>yI 	
 [
"}HHOOAqaO8;;%++;N	88??2r*--EKK-@88??2r*--EKK-@r>   c                     U R                  U5      nU R                  U5      nUR                  SS5      R                  5       nU R	                  U5      nU R                  U5      nU R                  U5      $ )Nr   rK  )r  r{  r   rv  r   rI  r  r%  s     r;   r
   ManualConvLinearQATModel.forwardy	  s\    JJqMIIaLFF2rN%%'HHQKHHQK||Ar>   )r{  r  r   rI  r  r  rM   r1  r   s   @r;   rI  rI  g	  s    A r>   rI  c                   0   ^  \ rS rSrSrSU 4S jjrSrU =r$ )ManualConvLinearSymmQATModeli	  z`Same as ManualConvLinearQATModule but with Symmetric Quantization.
Supported only with qnnpack.
c                 ,   > [         TU ]  [        5        g rM   )r  r<   r   r!  s    r;   r<   %ManualConvLinearSymmQATModel.__init__	  s    >?r>   rd   r.  )r]   r^   r_   r`   ra   r<   rc   r  r   s   @r;   rP  rP  	  s    @ @r>   rP  c                      ^  \ rS rSrSU 4S jjr  S	S\R                  S\\R                     S\\R                     4S jjrSr	U =r
$ )
ManualEmbeddingBagLineari	  c                 L  > [         TU ]  5         [        R                  " SSSS9U l        [
        U R                  l        [        5       U l        [        5       U l
        [        R                  " SS5      R                  [        R                  S9U l        [!        S5      U l        g )Nrs      r   )r  r  r  rt   r   r:  )r  r<   r   r  embr	   r  r   r  r   r  r  r   rh   r   r*  r   r!  s    r;   r<   !ManualEmbeddingBagLinear.__init__	  sp    ??"BUS8[
"}iiA&)))<.y9r>   r  r  per_sample_weightsc                     U R                  XU5      nU R                  U5      nU R                  U5      nU R                  U5      $ rM   )rW  r  r*  r  )r:   r  r  rY  r  s        r;   r
   ManualEmbeddingBagLinear.forward	  s=     HHU%78JJqMKKN||Ar>   )r  rW  r*  r  r  r.  r  )r]   r^   r_   r`   r<   rh   r  r/   r
  rc   r  r   s   @r;   rT  rT  	  sL    : +/59		||	 %,,'	 %U\\2		 	r>   rT  c                   j   ^  \ rS rSrSrSU 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )	DeFusedEmbeddingBagLineari	  zA module to simulate QAT embedding bag with a linear layer,
this module uses a separate embedding and bagging op, similar
to that which is described in the EmbeddingBag documentation.

https://pytorch.org/docs/stable/generated/torch.nn.EmbeddingBag.html
r  c                 t  > [         TU ]  5         [        R                  " SSS9U l        [
        U R                  l        [        R                  U l	        [        5       U l        [        5       U l        [        R                  " SS5      R                  [        R                   S9U l        [%        S5      U l        g )Nrs   rV  r  rt   r   r:  )r  r<   r   r  rW  r	   r  rh   r   
bagging_opr   r  r   r  r  r   r   r*  r   r!  s    r;   r<   "DeFusedEmbeddingBagLinear.__init__	  sy    <<rD8))[
"}iiA&)))<.y9r>   r  c                     U R                  U R                  U5      SS9nU R                  U5      nU R                  U5      nU R	                  U5      $ Nrt   r  )r_  rW  r  r*  r  )r:   r  r  s      r;   r
  !DeFusedEmbeddingBagLinear.forward	  sD    OODHHUOO3JJqMKKN||Ar>   )r_  r  rW  r*  r  r  r.  )r]   r^   r_   r`   ra   r<   rh   r  r
  rc   r  r   s   @r;   r]  r]  	  s-    :U\\ ell  r>   r]  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )SubModelForFusioni	  c                    > [         TU ]  5         [        R                  " SSSS S9R	                  [
        R                  S9U l        [        R                  " S5      R	                  [
        R                  S9U l	        g )Nrp  rt   ry  r   )
r  r<   r   rz  r   rh   r   r{  r  r  r!  s    r;   r<   SubModelForFusion.__init__	  sX    IIaAD1445;;4G	..#&&U[[&9r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   r  r%  s     r;   r
  SubModelForFusion.forward	  r  r>   r  r.  r^  r   s   @r;   re  re  	  s    :
 r>   re  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )SubModelWithoutFusioni	  c                    > [         TU ]  5         [        R                  " SSSS S9R	                  [
        R                  S9U l        [        R                  " SS9R	                  [
        R                  S9U l	        g )Nrp  rt   ry  r   Fr   )
r  r<   r   rz  r   rh   r   r{  r  re  r!  s    r;   r<   SubModelWithoutFusion.__init__	  sX    IIaAD1445;;4G	GGE*--EKK-@	r>   c                 B    U R                  U R                  U5      5      $ rM   )re  r{  r%  s     r;   r
  SubModelWithoutFusion.forward	  s    yy1&&r>   )r{  re  r.  r^  r   s   @r;   rk  rk  	  s    A
' 'r>   rk  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )ModelForFusioni	  c                   > [         TU ]  5         [        R                  " SSSS S9R	                  [
        R                  S9U l        [        R                  " S5      R	                  [
        R                  S9U l	        [        R                  " SS9R	                  [
        R                  S9U l        [        5       U l        [        5       U l        [        R                   " SS	5      R	                  [
        R                  S9U l        [%        5       U l        [)        5       U l        Xl        [        R.                  " SSS
S S9R	                  [
        R                  S9U l        [        R                  " SS9R	                  [
        R                  S9U l        [        R4                  " S5      R	                  [
        R                  S9U l        [        R                  " SS9R	                  [
        R                  S9U l        [        R:                  " SSS5      R	                  [
        R                  S9U l        [        R>                  " S5      R	                  [
        R                  S9U l         [        R                  " SS9R	                  [
        R                  S9U l!        S U R                  l        S U R"                  l        g )Nr  rp  rt   ry  r   Tr   $   rs   )rt   rt   rt   F)"r  r<   r   rz  r   rh   r   r  r  bn1r  r  re  r|  rk  r}  r  r  r   r  r   r  r  Conv3dr  r  BatchNorm3dbn2relu3Conv1dconv3r+  bn3relu4rD  s     r;   r<   ModelForFusion.__init__	  s   YYq!QT255EKK5H
>>!$''ekk':WWT*--EKK-@
%'	)+	))B#&&U[[&9[
"}YYq!YT:==EKK=P
WWU+..U[[.A
>>!$''ekk':WWT*--EKK-@
YYq!Q'***=
>>!$''ekk':WWT*--EKK-@
 		r>   c                    UR                  S5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nUR                  S5      nUR                  S5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nUR                  SS5      R                  5       nU R                  U5      nU R                  U5      nU R!                  U5      nU R#                  U5      nU R%                  U5      nU R                  U5      nU$ )Nrp  r   rs  )r  r  rz  r{  r|  r  r  rt  r  r|  r  r}  r   rv  r  r  r  rw  rx  r:   r  ys      r;   r
  ModelForFusion.forward	  s   IIaLJJqMJJqMHHQKJJqMKKNKKNJJqMHHQKJJqMIIaLLLOIIaLIIb"((*GGAJJJqMJJqMHHQKJJqMLLOr>   )rt  rw  r{  r  r  rz  r  r  r  r  r  r  rx  r|  r|  r}  r^  r   s   @r;   rq  rq  	  s    , r>   rq  c                   ,   ^  \ rS rSrSU 4S jjrSrU =r$ )
ConvBNReLUi	  c           
         > [         TU ]  [        R                  " SSSSSS9[        R                  " S5      [        R
                  " SS95        g )Nr  rt   Fry  r   )r  r<   r   rz  r  r  r!  s    r;   r<   ConvBNReLU.__init__ 
  s;    IIaAqu-r~~a/@"''RWBX	
r>   rd   r.  )r]   r^   r_   r`   r<   rc   r  r   s   @r;   r  r  	  s    
 
r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )ModelWithSequentialFusioni
  c                   > [         TU ]  5         [        R                  " SSS5      U l        [        R
                  " SS9U l        [        S5       Vs/ s H  n[        5       PM     nn[        R                  " U6 U l
        [        R                  " SS5      [        R
                  " SS9/n[        R                  " U6 U l        [        R                  " 5       U l        [        5       U l        [!        5       U l        g s  snf )Nr  rt   Fr   ,  rs   )r  r<   r   rz  r  r  r  rx   r  r  featuresr  
classifierseqr   r  r   r  )r:   r   layersheadr  s       r;   r<   "ModelWithSequentialFusion.__init__
  s    YYq!Q'
WWU+
(-a11*,1v.		#r"BGGE$:;--.==?[
"} 2s   C=c                 "   U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      n[        R
                  " US5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ )N)r   r  )	r  r  r  r  rh   r   r  r  r  r%  s     r;   r
  !ModelWithSequentialFusion.forward
  su    JJqMJJqMJJqMMM!MM!./OOAHHQKLLOr>   )r  r  r  r  r  r  r  r.  r^  r   s   @r;   r  r  
  s    
%	 	r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )ModelForFusionWithBiasi
  c                 z  > [         TU ]  5         [        R                  " SSSSS9R	                  [
        R                  S9U l        [        R                  " S5      R	                  [
        R                  S9U l	        [        R                  " SS9R	                  [
        R                  S9U l        [        R                  " SSSSS9R	                  [
        R                  S9U l        [        R                  " S5      R	                  [
        R                  S9U l        [        5       U l        [!        5       U l        g )	Nr  rp  r   Try  r   r   rt   )r  r<   r   rz  r   rh   r   r  r  rt  r  r  r  rw  r   r  r   r  r!  s    r;   r<   ModelForFusionWithBias.__init__ 
  s    YYq!QT255EKK5H
>>!$''ekk':WWT*--EKK-@
YYq!QT255EKK5H
>>!$''ekk':[
"}r>   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU R                  U5      nU$ rM   )r  r  rt  r  r  rw  r  r%  s     r;   r
  ModelForFusionWithBias.forward*
  sb    JJqMJJqMHHQKJJqMJJqMHHQKLLOr>   )rt  rw  r  r  r  r  r  r.  r^  r   s   @r;   r  r  
  s    % r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )ModelForLinearBNFusioni5
  c                 \  > [         TU ]  5         [        R                  " SS5      U l        [        R
                  " S5      U l        [        R                  R                  U R                  R                  5        [        R                  R                  U R                  R                  5        g )N   rs   )r  r<   r   r  r  r+  r  inituniform_r"  r#  r!  s    r;   r<   ModelForLinearBNFusion.__init__6
  sd    ))B#..$
(
&r>   c                 B    U R                  U R                  U5      5      $ rM   r  r  r%  s     r;   r
  ModelForLinearBNFusion.forward=
  s    wwtwwqz""r>   r  r.  r^  r   s   @r;   r  r  5
  s    '# #r>   r  c                        \ rS rSrS rS rSrg)DummyObserveriA
  c                     g)N)g      ?r   rd   rO   s    r;   calculate_qparamsDummyObserver.calculate_qparamsB
  s    r>   c                     U$ rM   rd   r%  s     r;   r
  DummyObserver.forwardE
  s    r>   rd   N)r]   r^   r_   r`   r  r
  rc   rd   r>   r;   r  r  A
  s    r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )ModelForConvTransposeBNFusioniI
  c                 r  > [         TU ]  5         [        R                  " SSS5      U l        [        R
                  " S5      U l        [        R                  " SSS5      U l        [        R                  " S5      U l
        [        R                  " SSS5      U l        [        R                  " S5      U l        g )Nr  rt   )r  r<   r   ConvTranspose1dr  r+  rt  r  r  r  rw  ConvTranspose3drz  rv  r{  r!  s    r;   r<   &ModelForConvTransposeBNFusion.__init__J
  s    ''1a0
>>!$''1a0
>>!$''1a0
>>!$r>   c                    U R                  U5      nU R                  U5      nUR                  S5      nU R                  U5      nU R	                  U5      nUR                  S5      nU R                  U5      nU R                  U5      nU$ )Nrp  )r  rt  r  r  rw  rz  r{  r%  s     r;   r
  %ModelForConvTransposeBNFusion.forwardS
  so    JJqMHHQKKKNJJqMHHQKKKNJJqMHHQKr>   )rt  rw  r{  r  r  rz  r.  r^  r   s   @r;   r  r  I
  s    %	 	r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )ModelWithFunctionalsi_
  c                    > [         TU ]  5         [        R                  " 5       U l        [        R                  " 5       U l        [        R                  " 5       U l        [        R                  " 5       U l        g rM   )r  r<   r  FloatFunctionalmycatmyadd
myadd_relumymatmulr!  s    r;   r<   ModelWithFunctionals.__init__`
  sN    ((*
((*
--/++-r>   c                     U R                   R                  XU/5      nU R                  R                  X"5      nU R                  R                  X35      nU R                  R                  XDR                  5      nU$ rM   )	r  r  r  r  r  add_relur  matmulT)r:   r  r  zr  us         r;   r
  ModelWithFunctionals.forwardk
  s]    JJNNA!9%JJNN1 OO$$Q*MM  CC(
 r>   )r  r  r  r  r.  r^  r   s   @r;   r  r  _
  s    .	 	r>   r  c                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )
ResNetBaseiw
  c                 
  > [         TU ]  5         [        R                  nSn[        R                  " X"SSS9U l        U" U5      U l        [        R                  " 5       U l        [        R                  " 5       U l	        [        R                  R                  5       U l        [        R                  R                  5       U l        [        R                   " S5      U l        [        R                  R%                  US5      U l        g )Nr  r  Fry  rt   )r  r<   r   r  rz  r  rt  r  r  r  rh   Identity
downsampler$  r  myopAdaptiveAvgPool2davgpoolr  r  r:   
norm_layerinplanesr  s      r;   r<   ResNetBase.__init__x
  s    ^^
YYx6F
h'WWY
WWY
((++-LL002	++F3((//(A.r>   c                 X   U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  R                  X#5      nU R                  U5      nU R                  U5      n[        R                  " US5      nU R                  U5      nU$ )Nrt   )r  rt  r  r  r  r  r  r  rh   flattenr  r:   r  r  identitys       r;   r
  ResNetBase.forward
  s    jjmhhsmjjo??1%iimmC*jjoll3mmC#ggcl
r>   c                     U R                   (       a-  [        R                  R                  R	                  U / SQ/SS9  g [        R                  R                  R                  U / SQ/SS9  g )N)r  rt  r  Tr   r  rO   s    r;   r  ResNetBase.fuse_model
  s[    ==HH!!22014 3  HH!!..014 / r>   )r  rt  r  r  r  r  r  r  r.  )	r]   r^   r_   r`   r<   r
  r  rc   r  r   s   @r;   r  r  w
  s    /
	 	r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )ModelMultipleOpsi
  c                 t  > [         TU ]  5         [        R                  nSn[        R                  " X"SSS9U l        [        R                  " X"SSS9U l        U" U5      U l        [        R                  " 5       U l	        [        R                  " 5       U l
        [        R                  R                  5       U l        [        R                  R                  5       U l        [        R                  R                  5       U l        [        R$                  " S5      U l        [        R(                  " SS5      U l        g Nr  r  Fry  )r   r   rV     )r  r<   r   r  rz  r  r  rt  r  r  r  rh   r  r  r$  r  skip_addr  r  r  r  r  r  s      r;   r<   ModelMultipleOps.__init__
  s    ^^
YYx6F
YYx6F
h'WWY
WWY
((++-446<<//1++F3))B"r>   c                    U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  R                  X#5      nU R                  U5      nU R                  U5      nU R                  U5      n[        R                  R                  R                  USS5      nU R                  R                  X"/5      nUR                  SS5      nU R                  U5      nU$ Nrp  r   rV  )r  rt  r  r  r  r  r  r  r  rh   r   
functional
max_pool2dr  r   r  r  s       r;   r
  ModelMultipleOps.forward
  s    jjmhhsmjjo??1%mm.jjoll3jjohh!!,,S!Q7hhllC:&kk"i(ggcl
r>   )
r  rt  r  r  r  r  r  r  r  r  r.  r^  r   s   @r;   r  r  
  s    # r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )ModelMultipleOpsNoAvgPooli
  c                 .  > [         TU ]  5         [        R                  nSn[        R                  " X"SSS9U l        [        R                  " X"SSS9U l        U" U5      U l        [        R                  " 5       U l	        [        R                  " 5       U l
        [        R                  R                  5       U l        [        R                  R                  5       U l        [        R                  " S5      U l        [        R"                  " SS5      U l        g r  )r  r<   r   r  rz  r  r  rt  r  r  r  r$  r  r  r  	MaxPool2dmaxpoolr  r  r  s      r;   r<   "ModelMultipleOpsNoAvgPool.__init__
  s    ^^
YYx6F
YYx6F
h'WWY
WWY
446<<//1||F+))B"r>   c                    U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R                  R                  X#5      nU R                  U5      nU R                  U5      nU R                  U5      n[        R                  R                  R                  USS5      nU R                  R                  X"/5      nUR                  SS5      nU R                  U5      nU$ r  )r  rt  r  r  r  r  r  r  rh   r   r  r  r  r   r  )r:   r  r  skips       r;   r
  !ModelMultipleOpsNoAvgPool.forward
  s    jjmhhsmjjozz!}mm*jjoll3jjohh!!,,S!Q7hhllC:&kk"i(ggcl
r>   )	rt  r  r  r  r  r  r  r  r  r.  r^  r   s   @r;   r  r  
  s    # r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )EmbeddingBagModulei
  c                 n   > [         TU ]  5         [        R                  R	                  SSSSSS9U l        g )Nrs   rV  TFr   r  )r  r<   rh   r   r  rW  r!  s    r;   r<   EmbeddingBagModule.__init__
  s8    88(( $$ ) 
r>   c                 &    U R                  XU5      $ rM   rW  )r:   r  r  rY  s       r;   r
  EmbeddingBagModule.forward
  s    xx*<==r>   r  r.  r^  r   s   @r;   r  r  
  s    
> >r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )EmbeddingModulei
  c                 h   > [         TU ]  5         [        R                  R	                  SSS9U l        g Nrs   rV  r  r  r<   rh   r   r  rW  r!  s    r;   r<   EmbeddingModule.__init__
  s)    88%%Rr%Jr>   c                 $    U R                  U5      $ rM   r  r:   r  s     r;   r
  EmbeddingModule.forward
  s    xx  r>   r  r.  r^  r   s   @r;   r  r  
  s    K! !r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )EmbeddingWithStaticLineari
  c                 .  > [         TU ]  5         [        R                  R	                  SSS9U l        [        R                  R                  SS5      U l        [        U R
                  l	        [        U l	        [        5       U l        [        5       U l        g )Nrs   rV  r  r   rp  )r  r<   rh   r   r  rW  r  r  r   r  r   r   r  r   r  r!  s    r;   r<   "EmbeddingWithStaticLinear.__init__
  sd    88(("(M((//!Q'<&[
"}r>   c                     U R                  X5      nU R                  U5      nU R                  U5      nU R                  U5      n[        R
                  " U/U/-   SS9nU$ rb  )rW  r  r  r  rh   r  )r:   r  r  	linear_inrW  q_xr  r  s           r;   r
  !EmbeddingWithStaticLinear.forward  sV    hhw(jj#WWS\\\"99bTSE\q1r>   )r  rW  r  r  r  r.  r^  r   s   @r;   r  r  
  s    % r>   r  c                      ^  \ rS rSr  SU 4S jjrS\R                  S\R                  S\R                  4S jrSrU =r	$ )	DenseTopMLPi  r  c                   > [         TU ]  5         [        R                  " [        R                  " X5      5      U l        [        R                  " [        R                  " X#-   U5      [        R                  " XE5      5      U l        g rM   )r  r<   r   r  r  	dense_mlptop_mlp)r:   	dense_dim	dense_outr  
top_out_intop_out_outr  s         r;   r<   DenseTopMLP.__init__  s[     	IIi+
 }}IIi/<IIj.
r>   sparse_featuredensec                 ~    U R                  U5      n[        R                  " U/U/-   SS9nU R                  U5      nU$ rb  )r  rh   r  r  )r:   r  r  dense_featurer  r  s         r;   r
  DenseTopMLP.forward  s?    
 u-99m_/??QGll8$
r>   )r  r  r.  )
r]   r^   r_   r`   r<   rh   r  r
  rc   r  r   s   @r;   r  r    sB    
	
		 ||	 
		 	r>   r  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )EmbBagWrapperi'  c                 V   > [         TU ]  5         [        R                  " XSS9U l        g )Nr   )r  )r  r<   r   r  emb_bag)r:   r  r  r  s      r;   r<   EmbBagWrapper.__init__(  s     ~5Qr>   c                 $    U R                  X5      $ rM   r  )r:   r  r  s      r;   r
  EmbBagWrapper.forward,  s    ||G--r>   r  r^  r   s   @r;   r  r  '  s    R. .r>   r  c                      ^  \ rS rSrSrSrSrSrSrSr	Sr
SU 4S jjrS	\R                  S
\R                  S\R                  S\R                  4S jrSrU =r$ )SparseNNModeli0  rs   r   r   rp  rt   r  c                    > [         TU ]  5         [        U R                  U R                  5      U l        [        U R                  U R                  U R                  U R                  U R                  5      U l        g rM   )r  r<   r  _NUM_EMBEDDINGS_EMBEDDING_DIMmodel_sparser  
_DENSE_DIM_DENSE_OUTPUT_TOP_OUT_IN_TOP_OUT_OUT	dense_topr!  s    r;   r<   SparseNNModel.__init__9  s`    )$*>*>@S@ST$OO
r>   sparse_indicessparse_offsetsr  c                 J    U R                  X5      nU R                  XC5      nU$ rM   )r  r"  )r:   r$  r%  r  r  r  s         r;   r
  SparseNNModel.forwardE  s'     **>Jnn^3
r>   )r"  r  r.  )r]   r^   r_   r`   r  r  r  r  r   r!  _TOP_MLP_DIMr<   rh   r  r
  rc   r  r   s   @r;   r  r  0  sk    ONJMKLL

		 	 ||		
 
	 	r>   r  c                      \ rS rSr " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r	 " S S	\R
                  R                  5      r
 " S
 S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S S\R
                  R                  5      r " S  S!\R
                  R                  5      r " S" S#\R
                  R                  5      r " S$ S%\R
                  R                  5      r " S& S'\R
                  R                  5      r " S( S)\R
                  R                  5      r " S* S+\R
                  R                  5      r " S, S-\R
                  R                  5      r " S. S/\R
                  R                  5      r " S0 S1\R
                  R                  5      rS2rg3)4TestHelperModulesiQ  c            
           \ rS rSrS\R
                  S\R
                  S\R
                  S\R
                  S\R
                  4
S jrS rS	rg
)TestHelperModules.ControlFlowiR  xspred1pred2r  r  c           
      h  ^^^^	 S[         R                  S[         R                  4S jm	S[         R                  S[         R                  4S jmS[         R                  S[         R                  S[         R                  4UU	4S jjmS[         R                  S[         R                  4S jmS[         R                  S	[         R                  S[         R                  S[         R                  S[         R                  4
UU4S
 jjn[         R                  " XD5      n[        R                  " XQX#U5      $ )Nr  r  c                 :    X -   n [         R                  " X 5      n U $ rM   rh   mmr  s    r;   true_nested:TestHelperModules.ControlFlow.forward.<locals>.true_nestedZ  s    EHHQNr>   c                 .    [         R                  " X 5      $ rM   r2  r4  s    r;   false_nested;TestHelperModules.ControlFlow.forward.<locals>.false_nested_  s    xx~%r>   r  r/  c                 @   > [         R                  " UTTU /5      nX-   $ rM   )r   cond)r  r/  r  r8  r5  s      r;   true_fn6TestHelperModules.ControlFlow.forward.<locals>.true_fnb  s"     %%e[,Lur>   c                 "    U R                  5       $ rM   )cos)r  r   s     r;   false_fn7TestHelperModules.ControlFlow.forward.<locals>.false_fnf  s    uuwr>   r.  c                    > U R                  5       n [        R                  " UTTX2/5      nX-   n U R                  5       $ rM   )r?  r   r;  sin)r  r.  r/  r  r@  r<  s       r;   map_fn5TestHelperModules.ControlFlow.forward.<locals>.map_fni  s:     EEG %%eWh
KEuuwr>   )rh   r  r3  r   rv  )
r:   r-  r.  r/  r  rD  r@  r8  r<  r5  s
         @@@@r;   r
  %TestHelperModules.ControlFlow.forwardS  s    u||  
& & &5<<    ELL  	<<	||	 ||	 <<		
 	 	 A##Fa@@r>   c                     [         R                  " SS5      [         R                  " S/5      [         R                  " S/5      [         R                  " SS5      4$ )Nrp  F)rh   onesr   rO   s    r;   r  ,TestHelperModules.ControlFlow.example_inputsw  sD    

1a eW%eW%

1a 	 r>   rd   N)	r]   r^   r_   r`   rh   r  r
  r  rc   rd   r>   r;   ControlFlowr,  R  sW    "	A"	A <<"	A <<	"	A
 ||"	A \\"	AH	r>   rJ  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )%TestHelperModules.Conv2dPropAnnotatoni  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R                  SS5      U l        g Nr  )r  r<   rh   r   rz  r{  r  r*  r!  s    r;   r<   .TestHelperModules.Conv2dPropAnnotaton.__init__  s<    G1a0DI((//!Q/DKr>   c                     U R                  U5      nUR                  SS5      n[        R                  R                  R                  USS5      nU R                  U5      nU$ )Nr   r  g      r<  )r{  r   rh   r   r  hardtanhr*  r%  s     r;   r
  -TestHelperModules.Conv2dPropAnnotaton.forward  sN    		!Ar1A##,,Qc:AAAHr>   )r{  r*  r.  r^  r   s   @r;   Conv2dPropAnnotatonrL    s    	0
	 	r>   rS  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ ))TestHelperModules.Conv2dWithObsSharingOpsi  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R                  5       U l        [        R                  R                  S5      U l	        g Nr  r  )
r  r<   rh   r   rz  r{  HardtanhrQ  r  adaptive_avg_pool2dr!  s    r;   r<   2TestHelperModules.Conv2dWithObsSharingOps.__init__  sQ    G1a0DI!HH--/DM',xx'A'A&'ID$r>   c                     U R                  U5      nU R                  U5      nU R                  U5      n[        R                  " U5      nU$ rM   )r{  rY  rQ  rh   meanr%  s     r;   r
  1TestHelperModules.Conv2dWithObsSharingOps.forward  s?    		!A((+Aa A

1AHr>   )rY  r{  rQ  r.  r^  r   s   @r;   Conv2dWithObsSharingOpsrU    s    	J	 	r>   r^  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ ),TestHelperModules.Conv2dWithTwoLinearPermutei  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R                  SSSS9U l        [        R                  R                  SS5      U l        g )Nr     rG  Fry  	r  r<   rh   r   rz  r{  r  r  r  r!  s    r;   r<   5TestHelperModules.Conv2dWithTwoLinearPermute.__init__  V    G2q1DI 88??2qu?=DL 88??1a0DLr>   c                     U R                  U5      n[        R                  " US5      nU R                  U R	                  U5      5      $ Nr   rp  r  rt   )r{  rh   permuter  r  r:   r  conv_outpermute_outs       r;   r
  4TestHelperModules.Conv2dWithTwoLinearPermute.forward  s7    yy|H--,?K<<[ 9::r>   r{  r  r  r.  r^  r   s   @r;   Conv2dWithTwoLinearPermuter`        	1	; 	;r>   ro  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )%TestHelperModules.Conv2dWithTwoLineari  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R                  SSSS9U l        [        R                  R                  SS5      U l        g )Nr  rb  rK  rG  Fry  rc  r!  s    r;   r<   .TestHelperModules.Conv2dWithTwoLinear.__init__  re  r>   c                     U R                  U5      n[        R                  " US5      nU R                  U R	                  U5      5      $ )N)rp  rK  )r{  rh   r   r  r  )r:   r  rk  reshape_outs       r;   r
  -TestHelperModules.Conv2dWithTwoLinear.forward  s7    yy|H--':K<<[ 9::r>   rn  r.  r^  r   s   @r;   Conv2dWithTwoLinearrr    rp  r>   rx  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )$TestHelperModules.ConvLinearWPermutei  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R                  SS5      U l        g )Nr  rG  )r  r<   rh   r   rz  r{  r  r  r!  s    r;   r<   -TestHelperModules.ConvLinearWPermute.__init__  s<    G1a0DI 88??1a0DLr>   c                 t    U R                  U5      n[        R                  " US5      nU R                  U5      $ rg  )r{  rh   ri  r  rj  s       r;   r
  ,TestHelperModules.ConvLinearWPermute.forward  s.    yy|H--,?K<<,,r>   )r{  r  r.  r^  r   s   @r;   ConvLinearWPermuterz    s    	1
	- 	-r>   r  c                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )!TestHelperModules.TwoLinearModulei  c                    > [         TU ]  5         [        R                  R	                  SSSS9U l        [        R                  R	                  SS5      U l        g )NrG  rb  Fry  )r  r<   rh   r   r  r  r  r!  s    r;   r<   *TestHelperModules.TwoLinearModule.__init__  s>    G 88??1bu?=DL 88??2q1DLr>   c                 B    U R                  U R                  U5      5      $ rM   )r  r  r%  s     r;   r
  )TestHelperModules.TwoLinearModule.forward  s    <<Q00r>   c                 2    [         R                  " SS5      4$ )Nrp  rG  rh   randnrO   s    r;   r  0TestHelperModules.TwoLinearModule.example_inputs  s    KK1%''r>   r  r.  	r]   r^   r_   r`   r<   r
  r  rc   r  r   s   @r;   TwoLinearModuler    s    	2
	1	( 	(r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )TestHelperModules.ConvMaxPool2di  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R                  SS5      U l        g )Nrp  rt   )r  r<   rh   r   rz  r{  r  poolr!  s    r;   r<   (TestHelperModules.ConvMaxPool2d.__init__  s>    G1a0DI**1a0DIr>   c                 J    U R                  U5      nU R                  U5      nU$ rM   r{  r  r%  s     r;   r
  'TestHelperModules.ConvMaxPool2d.forward  s!    		!A		!AHr>   r  r.  r^  r   s   @r;   ConvMaxPool2dr    s    	1
	 	r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )+TestHelperModules.ConvWithAdaptiveAvgPool2di  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R                  S5      U l        g rW  )r  r<   rh   r   rz  r{  r  rY  r!  s    r;   r<   4TestHelperModules.ConvWithAdaptiveAvgPool2d.__init__  s=    G1a0DI',xx'A'A&'ID$r>   c                 J    U R                  U5      nU R                  U5      nU$ rM   )r{  rY  r%  s     r;   r
  3TestHelperModules.ConvWithAdaptiveAvgPool2d.forward  s$    		!A((+AHr>   )rY  r{  r.  r^  r   s   @r;   ConvWithAdaptiveAvgPool2dr    s    	J
	 	r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ ) TestHelperModules.ConvWithBNRelui  c                   > [         TU ]  5         [        R                  R                  [        R                  R
                  [        R                  R                  S.n[        R                  R                  [        R                  R                  [        R                  R                  S.nXb   " SSSXES9U l
        U(       a  Xr   " S5      U l        O#[        R                  R                  5       U l        U(       a$  [        R                  R                  5       U l        g [        R                  R                  5       U l        g )Nr  r  )r#  r  )r  r<   rh   r   ry  rz  ru  r+  r  rv  r{  r  r  r  re  )	r:   re  rr  r  r#  r  convsbnsr  s	           r;   r<   )TestHelperModules.ConvWithBNRelu.__init__  s    GEHHOOPE88''88''88''C
 
1aGDI(1+((++-!HHMMO	!HH--/	r>   c                 h    U R                  U5      nU R                  U5      nU R                  U5      $ rM   r  r%  s     r;   r
  (TestHelperModules.ConvWithBNRelu.forward  s*    		!A
A99Q<r>   r  )rp  TTr   r^  r   s   @r;   ConvWithBNRelur    s    	0&	  	 r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )!TestHelperModules.ConvTWithBNRelui  c                 &  > [         TU ]  5         [        R                  R                  [        R                  R
                  S.n[        R                  R                  [        R                  R                  S.nXR   " SSSUS9U l        U(       a  Xb   " S5      U l	        O#[        R                  R                  5       U l	        U(       a$  [        R                  R                  5       U l        g [        R                  R                  5       U l        g )N)rt   rp  r  ry  )r  r<   rh   r   r  r  r+  r  convtr  r  r  re  )r:   re  rr  r  r#  convtsr  r  s          r;   r<   *TestHelperModules.ConvTWithBNRelu.__init__  s    G11ehh6N6NOFhh**uxx/C/CDCQ148DJ(1+((++-!HHMMO	!HH--/	r>   c                 h    U R                  U5      nU R                  U5      nU R                  U5      $ rM   )r  r  re  r%  s     r;   r
  )TestHelperModules.ConvTWithBNRelu.forward  s*    

1A
A99Q<r>   )r  r  re  )rp  TTr^  r   s   @r;   ConvTWithBNRelur    s    	0	  	 r>   r  c                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )"TestHelperModules.Conv2dThenConv1di  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R                  SSS5      U l        g rN  )r  r<   rh   r   ry  r  rz  r  r!  s    r;   r<   +TestHelperModules.Conv2dThenConv1d.__init__  s>    G((//!Q2DK((//!Q2DKr>   c                 l    U R                  U5      nUR                  S5      nU R                  U5      nU$ r   )r  r  r  r%  s     r;   r
  *TestHelperModules.Conv2dThenConv1d.forward  s.    AA		!AAAHr>   c                 6    [         R                  " SSSS5      4$ r  r  rO   s    r;   r  1TestHelperModules.Conv2dThenConv1d.example_inputs  s    KK1a+--r>   )r  r  r.  r  r   s   @r;   Conv2dThenConv1dr    s    	3
		. 	.r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )TestHelperModules.Conv2dWithCati  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R	                  SSS5      U l        g rN  r  r<   rh   r   rz  r  r  r!  s    r;   r<   (TestHelperModules.Conv2dWithCat.__init__  >    GAq1DJAq1DJr>   c                 v    U R                  U5      nU R                  U5      n[        R                  " X/SS9nU$ rb  r  r  rh   r  )r:   r  r  r  s       r;   r
  'TestHelperModules.Conv2dWithCat.forward"  s3    

1A

1A		1&a(AHr>   r  r.  r^  r   s   @r;   Conv2dWithCatr    s    	2
	 	r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )"TestHelperModules.Conv2dWithTwoCati(  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R	                  SSS5      U l        g rN  r  r!  s    r;   r<   +TestHelperModules.Conv2dWithTwoCat.__init__)  r  r>   c                     U R                  U5      nU R                  U5      n[        R                  " X/SS9nX4-   n[        R                  " Xe/5      nU$ rb  r  r:   rH  rI  x3x4r  r  r  s           r;   r
  *TestHelperModules.Conv2dWithTwoCat.forward.  sJ    BBBB		2(*AA		1&!AHr>   r  r.  r^  r   s   @r;   Conv2dWithTwoCatr  (  s    	2
	 	r>   r  c                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )!TestHelperModules.Conv2dWithSpliti6  c                    > [         TU ]  5         [        R                  R	                  SSS5      U l        [        R                  R	                  SSS5      U l        g rN  r  r!  s    r;   r<   *TestHelperModules.Conv2dWithSplit.__init__7  r  r>   c                     U R                  U5      n[        R                  " USSS9u  p#[        R                  " X#/SS9nU$ )Nrp  rt   r  )r  rh   splitr  )r:   r  rH  rI  r  s        r;   r
  )TestHelperModules.Conv2dWithSplit.forward<  s:    

1A[[A1-FB		2(*AHr>   c                 6    [         R                  " SSSS5      4$ )Nrt   r  rb  r  rO   s    r;   r  0TestHelperModules.Conv2dWithSplit.example_inputsC      KK1b"-//r>   r  r.  r  r   s   @r;   Conv2dWithSplitr  6  s    	2
		0 	0r>   r  c                       \ rS rSrS rSrg)TestHelperModules.ThreeAddiF  c                     X-   nX4-   nXV-   nU$ rM   rd   r  s           r;   r
  "TestHelperModules.ThreeAdd.forwardG  s    AAAHr>   rd   Nr]   r^   r_   r`   r
  rc   rd   r>   r;   ThreeAddr  F  s    	r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )!TestHelperModules.EmbeddingModuleiM  c                 h   > [         TU ]  5         [        R                  R	                  SSS9U l        g r  r  r!  s    r;   r<   *TestHelperModules.EmbeddingModule.__init__N  s)    Gxx))2)NDHr>   c                 $    U R                  U5      $ rM   r  r  s     r;   r
  )TestHelperModules.EmbeddingModule.forwardR  s    88G$$r>   r  r.  r^  r   s   @r;   r  r  M  s    	O	% 	%r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )+TestHelperModules.EmbeddingConvLinearModuleiU  c                    > [         TU ]  5         [        R                  R	                  SSS9U l        [        R                  R                  SSS5      U l        [        R                  R                  SS5      U l	        g )Nrs   rG  r  rb  )rt   r  )
r  r<   rh   r   r  rW  rz  r{  r  r*  r!  s    r;   r<   4TestHelperModules.EmbeddingConvLinearModule.__init__V  sW    Gxx))1)MDH2v6DI((//"a0DKr>   c                    U R                  U5      n[        R                  " USS9n[        R                  " US5      nU R	                  U5      n[        R                  " US5      n[        R
                  " USS9nU R                  U5      $ )Nr   r  )r   r  rt   rp  rh  )rW  rh   r  ri  r{  r  r*  )r:   r  
embeddingsrk  s       r;   r
  3TestHelperModules.EmbeddingConvLinearModule.forward\  sm    '*J;Jz<@Jyy,H}}X|<H}}X15H;;x((r>   )r{  rW  r*  r.  r^  r   s   @r;   EmbeddingConvLinearModuler  U  s    	1	) 	)r>   r  c                       \ rS rSrS rSrg)TestHelperModules.AddInplaceAddie  c                     X-   nX-  nU$ rM   rd   r  s      r;   r
  'TestHelperModules.AddInplaceAdd.forwardf      AFAHr>   rd   Nr  rd   r>   r;   AddInplaceAddr  e      	r>   r  c                       \ rS rSrS rSrg)TestHelperModules.MulInplaceMulik  c                     X-  nX-  nU$ rM   rd   r  s      r;   r
  'TestHelperModules.MulInplaceMul.forwardl  r  r>   rd   Nr  rd   r>   r;   MulInplaceMulr  k  r  r>   r  c                       \ rS rSrS rSrg)TestHelperModules.AddMulScalariq  c                 .    US-   nUS-  nUS-  nUS-  nU$ rN  rd   r%  s     r;   r
  &TestHelperModules.AddMulScalar.forwardr  s+    AAAAFAFAHr>   rd   Nr  rd   r>   r;   AddMulScalarr  q  s    	r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )+TestHelperModules.ConvBnReLU2dAndLinearReLUiy  c                    > [         TU ]  5         [        R                  SS9U l        [
        R                  R                  SSSS9U l        [
        R                  R                  5       U l
        g )NT)re  r  rG  Fry  )r  r<   r*  r  conv_bn_relurh   r   r  r*  r  re  r!  s    r;   r<   4TestHelperModules.ConvBnReLU2dAndLinearReLU.__init__z  sO    G 1 @ @d @ KD((//!QU/;DKDIr>   c                 x    U R                  U5      n[        R                  " US5      nU R                  U5      nU$ rg  )r  rh   ri  r*  )r:   r  rl  
linear_outs       r;   r
  3TestHelperModules.ConvBnReLU2dAndLinearReLU.forward  s6    !!!$A--<8K[1Jr>   )r  r*  re  r.  r^  r   s   @r;   ConvBnReLU2dAndLinearReLUr  y  s    	(	 	r>   r  c                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )!TestHelperModules.GroupwiseConv2di  c                 l   > [         TU ]  5         [        R                  R	                  SSSSS9U l        g )Nr   r  rp  )r  )r  r<   rh   r   rz  r{  r!  s    r;   r<   *TestHelperModules.GroupwiseConv2d.__init__  s*    G1a:DIr>   c                 $    U R                  U5      $ rM   r~  r%  s     r;   r
  )TestHelperModules.GroupwiseConv2d.forward  s    99Q<r>   c                 6    [         R                  " SSSS5      4$ )Nrp  r   rs   r  rO   s    r;   r  0TestHelperModules.GroupwiseConv2d.example_inputs  r  r>   r~  r.  r  r   s   @r;   GroupwiseConv2dr    s    	;	 	0 	0r>   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )!TestHelperModules.LinearReluModeli  c                    > [         TU ]  5         [        R                  R	                  SS5      R                  [        R                  S9U l        [        R                  R                  5       U l	        g r  r  r!  s    r;   r<   *TestHelperModules.LinearReluModel.__init__  sG    Ghhooa+..U[[.ADGDIr>   c                 F    U R                  U R                  U5      5      nU$ rM   r
  r%  s     r;   r
  )TestHelperModules.LinearReluModel.forward  s    		$''!*%AHr>   r  r.  r^  r   s   @r;   r  r    s    	(
	 	r>   r  rd   N) r]   r^   r_   r`   rh   r   r  rJ  rS  r^  ro  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rc   rd   r>   r;   r*  r*  Q  s   +ehhoo +Zehhoo %((// 
;UXX__ 
;
;ehhoo 
;	-UXX__ 	-
(%((// 
(	 		EHHOO 	   2 %((//  *.588?? .
 
588?? 0%((// 0 588?? %%((// %)EHHOO )   uxx EHHOO 	0%((// 	0%((// r>   r*  )r   rM   )r   rb  )r       )ra   rh   'torch.ao.nn.intrinsic.quantized.dynamicr  r   	intrinsicr$  rX  r  torch.ao.nn.quantizedr  torch.ao.nn.quantized.dynamicr  torch.distributeddistributedr   torch.nntorch.nn.functionalr  r  functorch.experimentalr   torch.ao.nn.intrinsicr   torch.ao.quantizationr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   +torch.ao.quantization.quantization_mappingsr   r    r!   torch.jit.mobiler"   (torch.testing._internal.common_quantizedr#   $torch.testing._internal.common_utilsr$   r%   torch.ao.ns.fx.ns_typesr&   r'   !torch.ao.quantization.quantize_fxr(   r)   r*   r+   torch.fxr,   torch.fx.graphr-   r  ImportErrorrQ  r4  r  r   r&  typingr.   r/   r0   collections.abcr1   numpyrf  torch._dynamo_dynamorV  torch.testingr2   r4   rj   ro   CrossEntropyLoss_default_loss_fnr   r   r   r   r   r   r   r   r   r  r  r7  r>  rB  rG  rN  rS  rY  r^  torchvisionHAS_TORCHVISIONskipIfskip_if_no_torchvisionrd  r  rn  r  r  r  r  r  r  r  r1  r<  rD  rS  r`  rn  rv  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r(  r6  r@  rN  rV  ra  ri  rz  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r#  r+  r3  r:  rB  rI  rP  rT  r]  re  rk  rq  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r*  rd   r>   r;   <module>r4     s    7 7 7 # # , ,      / .      8  8 N IL  %#F   	 	  ' ' $  # ##7 #7L 88,,.  3C &.. .2" F!O,P5f$$($$$$$O "_)<>NO P ')hh 	*!Z'>2BH>8 H>V23 2p
#UXX__ 
##ehhoo ##EHHOO ##RYY # ehhoo %((// $ 
) 
)
) 
)) )$)%((// )$)%((// ))588?? )()ehhoo )")uxx )B) )#%((// #.ryy .#588?? # 588?? "#ehhoo ##EHHOO #"# #&#UXX__ #&#ehhoo #<@ <@@)EHHOO )) )$)uxx )&UXX__ ,%((// 588?? (ehhoo  uxx 4EHHOO  S%((// S6
#uxx 
#
1uxx 
11EHHOO 11uxx 1 0		 01bii 1")uxx )2
/UXX__ 
//%((// /.bii ./")) /" UXX__  "'ehhoo '(UXX__  588?? $EHHOO $%((// uxx 6@#; @ryy ,		 2			 	'BII ',RYY ,^
 
		 2RYY ,	#RYY 	#EHHOO BII ,588?? 0# #Luxx H >> >!ehhoo ! &")) 8.BII .BII BH H[X  F`  Os$   ,"j 4j jjjj