
    ȅi                      S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKrS SKrS SKrS SKJrJr  S SKJr  S SKJr  S SKJr  S SKJr  S SKJr  S S	KJr  S S
KJrJrJrJrJ r   S SK!J"r"J#r#J$r$J%r%J&r&J'r'  S SK(J)r)  S SK*r+S SK,r+S SK-J.s  J/r0  S SK1J2r2  S SK+J3r3  S SK4J5r5  S SK6J7r7J8r9J	r:J.r;  S SK<J=r=  S SK>J?r?  S SK@JArAJBrBJCrCJDrDJErEJFrFJGrGJHrHJIrI  S SKJJ8rK  S SKLJMrM  S SKNJOrOJPrPJQrQJRrR  S SKSJTrTJUrUJVrV  S SKWJXrXJYrYJZrZJ[r[  S SK\J]r]  S SK^J_r_J`r`  S SKaJbrbJcrcJdrdJereJfrfJgrg  S SKhJiri  S SKjJkrkJlrlJmrmJnrnJoroJprpJqrqJrrrJsrs  S SKtJuru  S SKvJwrw  S S KxJyry  S S!KzJ{r{  S S"K,J|r|  S S#K}J~r~Jr  S S$KJr  S S%KJr  S S&KJr  S'S(KJr  S'S)KJrJr  S'S*KJr  S'S+KJr  S'S,KJr  S-S.KJ8r8JrJr  S-S/KJrJr  S-S0KJr  S-S1KJr  S-S2KJr  S-S3KJr  S-S4KJrJr  S-S5KJr  S-S6KJr  S-S7KJrJr  S-S8KJr  S-S9KJr  S-S:K.JrJrJrJrJrJrJrJrJr  S-S;KJr  \(       a"  S S<KJrJrJr  S S=KaJr  S S>KJr  S S?KJr  S-S@KJr  \$" SA5      r\" SB5      r\(       d  \8GR                  " 5       (       d  SSC jrSSD jrOS SEKJrJr  \(       a1  S SKrS SFKJrJrJr  \ \\\   /\\+GR                     4   \\\   \4   r " SG SH\GR                  5      r\ " SI SJ5      5       rSSK jrSSL jr\" 5       r\GR                  r\GR                  r\GR                  r\	GR                  " \5      r\+GR                  GR                  \SM5      r\+GR                  GR                  \SN5      r\+GR                  GR                  \SO5      r\+GR                  GR                  \SP5      r\+GR                  GR                  \SQ5      rSSR jrSSS jrSST jrSSU jr\GR                  " S5      SSV j5       r\GR                  SSW j5       rSSX jr        SSY jr S     SS[ jjr      SS\ jr S     SS] jjrSSS^ jjr   S         SS` jjrSSa jr    SSb jr      SSc jr S       SSd jjr S   SSe jjr\GR                  SSf j5       r " Sg Sh\&SZSi9r " Sj Sk\%5      r        SSl jGr \" SmSn9        SSo j5       Gr " Sp Sq5      Gr " Sr Ss\5      Gr " St SuG\5      Gr          SSv jGr      SSw jGr SSxSxSxSy.                   SSz jjjGrSS{ jGr        SS| jGr	 S       SS} jjGr
G\ S4         SS~ jjGr\" S 5      Gr                  SS jGrSS jGrSS jGr        SS jGrSS jGr\" S_S9 " S S5      5       GrSS jGrG\ SZ4               SS jjGrG\ 4         SS jjGr      SS jGrG\ SSSZ4             SS jjGr      SS jGr            SS jGr            SS jGrSS jGr        SS jGr        SS jGrSS jGr SSS.         SS jjjGrg)    )annotationsN)ABCabstractmethod)defaultdict)AbstractContextManager)	dataclass)currentframe)count)
attrgetter)AnyOptionalTYPE_CHECKINGTypeVarUnion)Neveroverride	ParamSpecProtocol	TypedDictUnpack)mock)#min_cut_rematerialization_partition)fx)enable_python_dispatcher)compiled_autogradconfigloggingutils)get_interface_for_device)wrap_compiler_debug)	chromium_event_timedCompileEventLoggercountersdetect_fake_modedynamo_timedflatten_graph_inputsget_metrics_contextlazy_format_graph_codeset_feature_use)r   )!unwrap_tensor_subclass_parameters)aot_export_moduleGraphOutputNamemake_boxed_funcSerializableAOTDispatchCompiler)	code_hashFxGraphCacheoutput_code_log)BoxedDeviceIndexformat_default_skip_message#log_cudagraph_skip_and_bump_counterPlaceholderInfo)CustomPartitionerFn)"create_mapping_pre_post_grad_nodessave_args_for_compile_fx_inner)CompiledAOTICompiledFxGraphCompiledFxGraphConstantsWithGmget_expanded_dimsindex_expanded_dims
OutputCode)	cache_dir)		BoxedBoolcount_tangentsfresh_cacheget_all_devices	InputTypeis_gpushould_assume_input_aligned should_use_remote_fx_graph_cachetensor_is_aligned)FakeScriptObject)is_opaque_type)trace_structured)compile_time_strobelight_meta)GraphModule)free_unbacked_symbolsSymExprPrinter)FakeTensorProp)_WaitCounter)
OrderedSet   )aot_autograd)ShortenTraceback	SkipFrame)_use_lazy_graph_module)_PyTreeCodeGen)
has_triton   )r   distributed_autotunemetrics)get_wrapper_codegen_for_deviceinit_backend_registration)DebugContext)select_decomp_table)InductorError)joint_graph_passes)post_grad_passesview_to_reshape)pre_grad_passes)GraphLowering)get_device_typeIRNode)complex_memory_overlap)TritonBundler)	align_inputs_from_check_idxsclone_preserve_stridescopy_misaligned_inputs get_cloned_parameter_buffer_name%get_first_incompatible_cudagraph_node#maybe_get_suppress_shape_guards_ctxoutput_noderemove_unaligned_input_idxsshape_env_from_inputs)V)Callable	GeneratorSequence)_StrideExprStr)
OpOverload)Weights)ExternKernelNode_P_Tc                "    [         R                  $ N)dynamo_utilsidentityattrs    T/home/james-whalen/.local/lib/python3.13/site-packages/torch/_inductor/compile_fx.pytime_and_logr      s    $$$    c                     g r    )argskwargss     r   log_optimus_to_scubar      s    r   )r   r   )FQNGraphInputNameGraphSignaturec                       \ rS rSrSrSrSrSrg)FxCompileMode   r   rZ   rS   r   N)__name__
__module____qualname____firstlineno__NORMAL	SERIALIZE
SUBPROCESS__static_attributes__r   r   r   r   r      s    F IJr   r   c                  4    \ rS rSr% S\S'   S\S'   S\S'   Srg)	FxCompileConfig   r   modebool	use_asyncuse_progressiver   Nr   r   r   r   __annotations__r   r   r   r   r   r      s    
Or   r   c                    Sn [         R                  R                  U 5      nUc  [        [        R
                  SS5      $ SnSnUR                  5       R                  S5      (       a  SnUSS  nUR                  5       R                  S5      (       a  SnUSS  n UR                  5       n[        [        U   X#5      $ ! [         a    SS K
nUR                  " [        5      nUR                  S	UU S
R                  [        S [        R                    5       5      5      5        [         R                  R#                  U 5        [        [        R
                  SS5      s $ f = f)NTORCHINDUCTOR_FX_COMPILE_MODEFzprogressive+T   zasync+   r   z>Invalid value of %s for %s. Expected one of %s. Using default.z, c              3  8   #    U  H  n[        U5      v   M     g 7fr   )repr.0xs     r   	<genexpr>+_fx_compile_mode_default.<locals>.<genexpr>   s     H.GT!WW.G   )osenvirongetr   r   r   lower
startswithupperKeyErrorr   	getLoggerr   errorjoinsorted__members__pop)namevaluer   r   r   logs         r   _fx_compile_mode_defaultr      s+   *DJJNN4 E}}33UEBBIO{{}//bc
{{}))	ab	C}U3YPP C)		LIIfHm.G.GHHI		
 	

t}33UEBBCs   "B> >BE E c                     SS0/$ )Nmax_autotuneTr   r   r   r   _get_progression_configsr      s     
 r   
perf_hintspre_grad_graphspost_grad_graphscudagraph_static_inputsinductor_metricsc                    [         R                  R                  R                  5       n[	        [        U 5      5      nU(       a  UR                  (       d  U$ UR                  R                  $ r   )torch_guardsTracingContexttry_getlistrangefw_metadatastatic_input_indices)	num_fixedcontextfixeds      r   get_static_input_idxsr      sM    
 mm**224Gy!"E'--333r   c                B   U R                   R                  SS9S   n/ n[        UR                  S   [        R
                  R                  5      (       d  UR                  S   nOUR                  nU H  n[        U[        R
                  R                  5      (       a^  UR                  R                  S5      =nb@  [        U[        R                  5      (       a!  UR                  UR                  5       5        M  UR                  S 5        M     X!R                  S'   g )Noutputopr   valoriginal_output_strides)graph
find_nodes
isinstancer   r   r   Nodemetar   Tensorappendstride)gmrq   output_stridesoutput_node_argsr   r   s         r   record_original_output_stridesr     s    ((%%%215KNk&&q)588==99&++A.&++"vuxx}}--..;3--!!#**,/ !!$' # 3A./r   c                    U R                   R                  S[        R                  R                  R
                  S9 H0  n[        XR                  S   R                  5      n[        U5        M2     [        U 5        g )Ncall_functionr   targetr   )r   r   r   opshigher_orderinvoke_subgraphgetattrr   r   )_recursive_record_original_output_stridesr   )r   nodesubgraphs      r   r   r     sa    ##599#9#9#I#I $  2yy|2231(;	 #2&r   c           	     "   U R                   R                  S[        R                  R                  R
                  S9 H  n[        XR                  S   R                  5      nUR                   R                  SS9 H}  n[        [        UR                  S   5      5       Vs/ s H@  n[        UR                  S   U   [        R                  R                  5      (       d  M>  UPMB     snUR                  S'   M     [        U5        M     g s  snf )Nr   r   r   r   r   user_visible_output_idxs)r   r   r   r   r   r   r   r   r   r   lenr   r   r   r   *_recursive_record_user_visible_output_idxs)r   r   r   idxs       r   r   r   %  s    ##599#9#9#I#I $  2yy|223NN---:D !TYYq\!2353Cdiil3/? 35DII01 ; 	38<5s   !=D"Dc                 6    [         R                  " [        5      $ r   )dynamo_loggingget_step_loggerr   r   r   r   _step_loggerr   5  s    ))#..r   c                 &   [         R                  R                  5       (       an  [         R                  R                  R                  R
                  (       d:  [         R                  R                  5       S:  a  [        R                  " S5        g g g g )N)   r   zTensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.)	r   cudais_availablebackendsmatmul
allow_tf32get_device_capabilitywarningswarnr   r   r   _warn_tf32_disabledr  :  sf     	

!!##**55JJ,,.&8d	
 9 6 	"r   c           
       ^ [        U R                  SS9 VVs/ s H  u  p#UPM	     snn5      mTR                  [        U R                  SS9 VVs/ s H  u  p#UPM	     snn5      5        SU4S jjnUR                  R
                   GHr  nUR                  S:X  d  M  UR                  nUR                  S5      (       d  UR                  S5      (       d  MP  [        X5      (       d  Mb  [        U5      " U5      n[        U5      " U 5      n[        U[        5      (       a1  [        U[        5      (       a  UR                  UR                  L a  M  ORUR                  UR                  :X  a8  UR                  UR                  :X  a  [         R"                  " Xx5      (       a  GM  UR                  S5      (       a  SOSn	U" UR                  U	5      n
U	 U
 3nXl        [%        XU5        TR'                  U5        GMu     gs  snnf s  snnf )	a  
In aot_export_module (make_fx), we create get_attr nodes with name prefix
"_tensor_constant" and "_torchbind_obj". See Tracer.create_arg() in
torch/fx/_symbolic_trace.py

However, this might result in name collision if the original mod already
has a different buffer with the same name.

We resolve this potential name collision here by changing the target name
with a new number post fix.
Fremove_duplicatec                r  > SnU R                    H  nUR                  S:X  d  M  UR                  R                  U5      (       d  M7  [	        UR                  5      [	        U5      :  d  M[  UR                  R                  U5      S   nUR                  5       (       d  M  [        U[        U5      5      nM     T Hu  nUR                  U5      (       d  M  [	        U5      [	        U5      :  d  M5  UR                  U5      S   nUR                  5       (       d  M`  [        U[        U5      5      nMw     US-   $ )Nr   get_attrrZ   )	nodesr   r   r   r   splitisdigitmaxint)r   prefixir   post_fixkeyexisting_keyss         r   find_smallest_i0_resolve_name_collision.<locals>.find_smallest_i[  s    KKDww*$)?)?)G)Gt{{#c&k1#{{008<H''))3x=1   !C~~f%%s8c&k)"yy04H''))3x=1 ! 1ur   r
  _tensor_constant_torchbind_objN)r   zfx.Graphr  strreturnr  )rR   named_parametersupdatenamed_buffersr   r  r   r   r   hasattrr   r   rI   real_objdevicedtyper   equalsetattradd)modr   r   r   r  r   target_name	gm_targetmodel_targetr  new_idnew_target_namer  s               @r   _resolve_name_collisionr,  G  s    "33U3KLK)$KLM #*;*;U*;*ST*SYTD*STU  77j ++K))" !,,-=>>3,,";/3I%k237L)%566|-=>>!**l.C.CC  L$7$77OO|'9'99KK	88  ))*<== #% 
 %RXXv6F!'1O)KB3o.I - 	M Us   H
Hc                   SSK JnJn  [        X5        0 nU R	                  SS9 H  u  pgXuU'   U" UUUUR
                  S9  M     U R                  SS9 H  u  phXU'   U" UUUUR                  S9  M     UR                  R                  SS9n	/ n
U	 H  nUR                  nXR                  ;   a"  UR                  U   nU
R                  U5        M@  XR                  ;   aE  UR                  U   nU
R                  U5        [        X^   5      UR                  [!        U5      '   M  XR"                  ;   d   eU
R                  S 5        M     SSKJn  [)        UR                  R+                  5       R,                  S   5      n/ nUR.                  nUR0                  nUR2                  n[5        U5       Hl  u  nnS nU[7        U5      [7        U5      -   [7        U5      -   :  a,  [9        UR                  5      nUU;   a  UU   nOUU;   a  UU   nUR                  U5        Mn     U" UU
U[:        R<                  " 5       S 5      n[?        URA                  5       5      UR                  S	'   U$ )
Nr   )_assign_attr	_AttrKindFr  )	attr_kindplaceholderr   )_unliftmutated_named_buffers)!torch.export.unflattenr.  r/  r,  r  	PARAMETERr  BUFFERr   r   r   inputs_to_parametersr   inputs_to_buffersrl   r   rn   user_inputstorch.export._unliftr2  tuplerq   r   buffers_to_mutateuser_inputs_to_mutateoutput_tokens	enumerater   r,   pytreetreespec_leafrR   values)r&  r   graph_signaturer.  r/  
state_dictr   parambufferplaceholder_nodeslifted_inputsr   	node_nameparameter_namebuffer_namer2  outputsmutated_outputsbuffer_mutationsuser_input_mutationsr>  r   outr   unlifted_gms                            r   _unlift_graphrR    sq    ?C$OQJ++U+C 4))		
 D ))5)A!4&&		
 B ++}+=)+M "II	<<<,AA)LN  0;;;);;IFK  -&z'>? GG4[AB  ; ;;;;  & " -).rxx/C/C/E/J/J1/M)NGO&88*@@#11Mg&S6:%&-A)BBSEWWW"388,D''(.--,T2u% ' 
K 1;;K;R;R;T0UK,-r   Fc              #  &  #    [        S U R                  R                  SS9 5       5      n[        5       nU R                  5        HH  u  pEXB;   d  M  [	        U[
        R                  R                  5      (       d  M7  UR                  U5        MJ     U(       ak  U R                  R                  S[
        R                  R                  R                  S9 H+  nUR                  UR                  S   R                  5        M-     U S h  vN   g  N7f)Nc              3  8   #    U  H  oR                   v   M     g 7fr   )r   r   s     r   r   &_get_subgraph_names.<locals>.<genexpr>  s      5<Q<r   r
  r   r   r   r   )rR   r   r   named_childrenr   r   r   rM   r%  r   r   r   discardr   r   )r   skip_invoke_subgraphall_subgraph_namesfx_subgraph_names
child_namechild_moduler   s          r   _get_subgraph_namesr]    s      +5 5((---<5 + *4$&$5$5$7 
 +
%((..1
 1
 !!*- %8 HH''uyy'='='M'M ( 
D %%diil&9&9:

 !  s   AD'D=BD	D
Dc                P   [        SSSS9   [        R                  (       d  U sS S S 5        $ [        R                  n[        R                  n[        U 5       H&  n[        X5      n[        US5      n[        XU5        M(     [        XX#5      sS S S 5        $ ! , (       d  f       g = f)N_recursive_pre_grad_passesTpre_grad_pass_time_uslog_pt2_compile_eventdynamo_compile_column_usr   )
r%   r   use_pre_grad_passesadd_pre_grad_passesremove_pre_grad_passesr]  r   r_  r$  re   )r   example_inputs
add_passesremove_passessubgraph_namer   new_subgraphs          r   r_  r_    s     
$"!8

 ))
 
 //
5504Mr1H5hCLB|4	 5
 r:M
 
 
s   BA B
B%c                J  ^ ^ SU U4S jjn[        SSSS9   [        R                  (       d  T sS S S 5        $ [        [	        T T5      5      nU H  nU" U5        M     [        T 5      n[	        USS9 H  nXC;  d  M
  U" U5        M     UsS S S 5        $ ! , (       d  f       g = f)	Nc                N   > [        TU 5      n[        UT5      n[        TX5        g r   )r   _recursive_joint_graph_passesr$  )rj  r   rk  r   rX  s      r   _run_on_sub_graph_module?_recursive_joint_graph_passes.<locals>._run_on_sub_graph_module  s&    2}-4X?STM0r   rn  Tjoint_graph_pass_time_usra  FrX  )rj  r  r  None)r%   r   use_joint_graph_passesrR   r]  rb   )r   rX  ro  old_subgraph_namesrj  out_gms   ``    r   rn  rn    s    1 1
 
'"!;

 ,,
 
 ((;B@T(UV/M$]3 0 $B' 1eTM6(7 U 7
 
 
s   BAB=B
B"c                    [        SSSS9   [        R                  (       d
   S S S 5        g [        U 5       H  n[	        X5      n[        X15        M     [        X5        S S S 5        g ! , (       d  f       g = f)N_recursive_post_grad_passesTpost_grad_pass_time_usra  )r%   r   use_post_grad_passesr]  r   rx  rc   )r   is_inferencerj  r   s       r   rx  rx  6  sh    	%"!9

 **
 
 14Mr1H'? 5 	*
 
 
s   A(3A((
A6Tc                   SSK JnJnJnJnJn  U" XX#5      n	Uc  U	" 5       OSn
[        [        U	R                  R                  5      S   R                  S   5       VVs0 s H  u  pUR                  U_M     nnn/ n/ n0 nU R                  R                   H^  nUR                  U;   a  UR                  U5        M&  UR                  U   U:X  d  M;  UR                  S:w  d  MM  UR                  U5        M`     U HC  nSUR                  -   nU" U UUc  XUR                        OSU5        UUR                     UU'   ME     USSS2    Hb  nUR                  (       a3  UR                   H!  nUR                  U   U:X  a  M   SU S35       e   MG  U R                  R!                  U5        Md     U R#                  5         U	U4$ s  snnf )	a  
This function takes an GraphModule input "gm".
The gm will be split into 2 components,
  1) const_gm, which consists the subgraph of gm that can be constant folded.
  2) gm (being inplace modified,) which returns the graph after constant folding.

If an additional "lifted_constants" argument is passed in, we will assume the gm has
been lifted and run the transformation accordingly.

When a "skip_folding_node_fn" callback is passed, we will skip constant folding on
the nodes for which the callback returns True.

const_output_index is a mapping of corresponding node name from gm to the
output index of const_gm.
Returns (const_gm, const_output_index)
r   )CONST_MODULE_TAGMETA_TAG
MODULE_TAGreplace_node_with_constantrun_and_get_constant_graphNr  r1  _FOLDED_CONST_znode: z user not empty.) torch._inductor.constant_foldingr}  r~  r  r  r  r?  r;  r   r  r   r   r   r   r   users
erase_node	recompile)r   skip_constructorlifted_constant_namesskip_folding_node_fnr}  r~  r  r  r  const_gmconst_resultr   r   const_outputsto_erase_nodeto_replace_nodeconst_output_indexr   new_const_namens                       r   split_const_gmr  E  s   ,  *
3H "7!>8:DL #,E(..2F2F,G,K,P,PQR,S"T"T"T   MO99%""4(YYx $44M9Q  &	   )DII5" )0 49956		
 .;499-E>*   dd#::ZZvvh':5VvEU7VV5   HH% $ LLN'''Es   Gc                n   [         R                  R                  n[        UR                  R
                  UR                  R
                  UR                  R
                  UR                  R
                  /5      nU H  nU R                  R                  SUS9 H  n[        UR                  R                  SS 5      [         R                  5      (       d  M>  UR                  S   R                  [         R                   :X  d  Mk  UR                  S   R"                  R$                  S:X  d  M      g   M     g)Nr   r   r   r   TF)r   r   atenrR   mmdefaultaddmmbmmbaddbmmr   r   r   r   r   r   r"  float32r!  type)r   r  tf32_opsr   r   s        r   is_tf32_warning_applicabler    s    99>>DGGOOJJHHLL  		
H HH''?6'JD499==5u||DDIIe$**emm;IIe$++00F: K  r   c                   [        S U  5       5      n[        R                  (       aE  [        R                  (       a0  U(       d)  [        R                  S5        [        R                  " SS9$ [        R                  R                  (       a)  [        R                  S5        [        R                  " SS9$ [        R                  " 5       $ )z
For CPU backend, enable comprehensive padding causes some unit tests
fail due to changing number of generated kernels. Skip for now.
c              3     #    U  HE  n[        U[        R                  5      (       d  M$  [        UR                  R
                  5      v   MG     g 7fr   )r   r   r   rE   r!  r  )r   ts     r   r   6maybe_disable_comprehensive_padding.<locals>.<genexpr>  s3      '5!Au||9Tqxx}}~s
   #A&Az!Skip comprehensive padding on CPUF)comprehensive_paddingz;Skip comprehensive padding for use_runtime_constant_folding)anyr   disable_padding_cpur  perf_hint_loginfopatchaot_inductoruse_runtime_constant_folding
contextlibnullcontext)rg  has_gpus     r   #maybe_disable_comprehensive_paddingr    s      '5 G !!f&B&B7>?||%88				9	9I	
 ||%88%%''r   c                p    U (       d  U(       a  [         R                  " SS9$ [        R                  " 5       $ )z@
graph partition does not support cpp_wrapper and aot_mode yet.
F)graph_partition)r   r  r  r  )cpp_wrapperaot_modes     r   maybe_disable_graph_partitionr    s'     h||E22%%''r   c                   [        5          [        U5      nU(       d4  [        R                  R	                  SS9n[        XS9R                  " U6   O^U(       d  [        R                  " 5       O [        R                  R                  USS5      nU   [        XS9R                  " U6   SSS5        SSS5        U$ ! , (       d  f       N= f! , (       d  f       W$ = f)zq
If we can not detect fake mode from the context of inputs, create one.

The created fake mode will be returned.
Tallow_non_fake_inputs)r   r  N)r   r$   r   _subclassesFakeTensorModerP   	propagater  r  r   r  objectpropagate_dont_convert_inputs)r   rg  force_allow_non_fake_inputs	fake_modectxs        r   fake_tensor_propr    s     
"	#$^4	))88t8TI2.88.I 3 &&(ZZ&&y2I4P 
 r2PP#  
$    
$	#  s$   BCB:(C:
C	C
Cc                    [         R                  " U 5         [         R                  " 5       sS S S 5        $ ! , (       d  f       g = fr   )r   r  get_config_copy)config_patchess    r   get_patched_config_dictr    s(     
n	%%%' 
&	%	%s	   6
Ac               #     #    [         R                  (       a   [        [        5       SS9   S v   S S S 5        g S v   g ! , (       d  f       g = f7f)NF)dirdelete)r   force_disable_cachesrB   r?   r   r   r   with_fresh_cache_if_configr    s4     "" Y[7 87 	 87s   (A=A
AAc                      \ rS rSr% S\S'   S\S'   S\S'   S\S	'   S\S
'   S\S'   S\S'   S\S'   S\S'   S\S'   S\S'   Srg)_CompileFxKwargsi  zOptional[BoxedBool]
cudagraphsSequence[int]static_input_idxsr   is_backwardzOptional[int]graph_idr  r  r{  zOptional[bool]
layout_optz1Optional[Callable[[list[ExternKernelNode]], Any]]extern_node_serializerzOptional[BoxedDeviceIndex]boxed_forward_device_index
fx_wrapperr   Nr   r   r   r   r  r    sC    ##$$NMM ::r   r  )totalc                  .    \ rS rSr        SS jrSrg)_CompileFxCallablei  c                    g r   r   )selfr   rg  r   s       r   __call___CompileFxCallable.__call__  s    
 r   r   Nr   rM   rg  Sequence[InputType]r   Unpack[_CompileFxKwargs]r  r>   )r   r   r   r   r  r   r   r   r   r  r    s-     , +	
 
r   r  c                   UR                  SS 5        UR                  SS5        UR                  SS5        UR                  SS 5        UR                  SS5        UR                  SS5        UR                  S	S5        UR                  S
S 5        UR                  SS 5        UR                  SS 5        [        R                  " 5        nUR                  [        R
                  R                  R                  5       5        UR                  [        [        R                  5      5        UR                  [        R                  " SSSSSSS95        UR                  [        5       5        UR                  [        5       5        [        R                   " SUS   S9  [#        [$        SS9" U U40 UD6sS S S 5        $ ! , (       d  f       g = f)Nr  r  r   r  Fr  r  r  r{  r  r  r  compile_fx_innerinductor_compileTcompile_inductor#inductor_cumulative_compile_time_us)
phase_namerb  log_waitcounterwaitcounter_name_overriderc  )r  inductor)compiler_name)
setdefaultr  	ExitStackenter_contextr   r   _python_dispatch_disable_current_modesrW   dynamo_configuse_lazy_graph_moduler   r%   r  r_   r"   pt2_compiler    _compile_fx_inner)r   rg  r   stacks       r   r  r    s   
 lD)
)2.
mU+
j$'
mU+
lE*
ne,
2D9
lD)
.5 
			5EKK88OOQR2=3V3VWX%%"-&* $*<)N		
 	689LN+&&}-	
 ##4JO
 
' 
 		s   
C'F;;
G	zcompilation time (in seconds)r   c                4  ^)^* [         R                  n[        R                  (       a&  SSKJn  UR                  5       nUR                  5         [        R                  R                  R                  R                  5         [        R                  " U R                  5      S:X  a  U(       d|  SSKJn  SSKJn  UR)                  U 5        [        R*                  R,                  R/                  5       n[0        R2                  " SSU0UR4                  S9  [7        U R8                  5      $ UR;                  SS	5      n	[<        R?                  S
U	5        [A        X5      n
[C        [E        [G        [I        U R                  RJ                  5      5      5      RL                  S   [N        [P        45      (       d   SU R                   35       eURS                  S5      c&  [U        [        RV                  RX                  5      US'   [        RZ                  (       a  []        U U40 UD6  [^        R^                  " 5       n[a        5       n[c        5         [e        S S [g        U 5       5        5       5      n[i        SSSS9   [        Rj                  (       + =(       a`    [        Rl                  =(       d    U=(       a@    U(       + =(       a2    U=(       a)    [        Rn                  R                  Rp                  (       + n[        Rl                  nUn[s        SU5        [t        R?                  SUUUU[        Rj                  5        [w        U5       H\  u  nn[C        U[        Rx                  5      (       d  M'  [{        UR|                  R~                  5      (       d  MM  UU	;   d  MU  SUl@        M^     SnSnSm)Sn[        U 5      n[^        R                  " 5       nU(       a  [        R                  " XX*U5      u  nm)Ubz  Uu  nn[t        R?                  SU5        U(       a*  [        R                  " 5       n[t        R?                  S5        [        R                  " UUUUUURS                  SS5      US9u  nm)O[t        R?                  S5        [        Rn                  R                  Rp                  (       as  Ub   eT)b   e[        R                  " 5          [        XU
40 UD6nUc   e[        R                  " 5       u  nnUR                  U5         [        R                  " 5         GOT)b	  T)S   S:X  a@  Ub   e[t        R?                  ST)b  T)RS                  SS5      OS 5         [        XU
40 UD6nGOOT)S   S!:X  Ga  Ub   eUc   e[t        R?                  S"5        [        R                  " 5          [        XU
40 UD6nUc   e[^        R                  " 5       U-
  UlT        Uu  nnUUlU        UUlV        [        R                  " 5       u  nnUR                  U5         [        R                  " 5         Ub  [        U5      T)S#'   UR                  T)S$'   [t        R?                  S%U5        [        R                  " UUUUU5        O>T)S   S&:X  d   eUc   eUc   eUu  nn[t        R?                  S'U5        UUlU        UUlV        Uc   eUnT)b  T)S   OS(m*[0        R                  " S)T* 3T)=(       d    0 US*9  [0        R                  " S+T*UT)(       a  T)RS                  S,5      OST)(       a  T)RS                  S-5      OST)(       a  T)RS                  S5      OS.UUS/9  T)b  [        S0U*4S1 jU)4S2 jS39  UR                  UUU5        SSS5        [t        R?                  S4[^        R^                  " 5       U-
  5        [t        R                  [        R                  5      (       Ga  / n[        S5   R                  5        GH	  u  nn UR                  S65      n![        U!5      S7:  a  UR                  US8S9S9S9U /5        M@  [        U!5      S::  a  S6R                  U!SS; 5      OS6R                  U!SS< 5      n"U"R                  S=5      n#U#(       aF  [        U!5      S::  a7  U!S;S u  n$n%n&n'S6R                  U!SS; 5      n"UR                  U"U$U%U&U'U /5        M  U!S<S u  n%n&n'S6R                  U!SS< 5      n"UR                  U"S8U%U&U'U /5        GM     [t        R                  S>5        [t        R                  S?R                  S@SASBSCSDSE5      5        [t        R                  SF5        U H:  n([t        R                  S?R                  " U(6 5        [t        R                  SF5        M<     [        R                  R                  R                  R                  5         [        5       " [        R                  SGUS   (       a  SHOSI SJUSK    35        W$ ! [        [        4 a    e [         a3  n[        U[        5       5      R                  UR                  5      SeSnAff = f! [        R                  " 5         f = f! [         a3  n[        U[        5       5      R                  UR                  5      SeSnAff = f! [        [        4 a    e [         a3  n[        U[        5       5      R                  UR                  5      SeSnAff = f! [        R                  " 5         f = f! , (       d  f       GN= f)Lz
Inductor API that compiles a single graph.

If you change the argument list for this function, make sure you
also update the call to save_args_for_compile_fx_inner below accordingly.
r   )AutotuneProcessPool)CompileEventLogLevel)_LazyGraphModulezbackward no-op
compile_id)metadata	log_levelr  r   z&static input idxs compile_fx_inner: %szGinductor can only compile FX graphs which return a tuple/list, but got r  Nc              3  D   #    U  H  nUc  M  UR                   v   M     g 7fr   )supports_caching)r   backends     r   r   $_compile_fx_inner.<locals>.<genexpr>|  s(      	#
G  	!  
s     c              3     #    U  H8  n[        UR                  [        R                  [        R                  5      v   M:     g 7fr   )r]   r  r   r  r  r   r!  s     r   r   r  ~  s<      
 . +V//1B1B  .s   A Afx_codegen_and_compileT)rb  r  fx_cachezXFX cache status: use_cache=%s, local=%s, remote=%s, aot_mode=%s, force_disable_caches=%szFX cache key generated: %szUsing remote FX cacher  F)r  	constantszFailed to generate FX cache keycache_statebypasszFX cache bypass reason: %scache_bypass_reasonunknownz*FX cache disabled or key generation failedmissz,FX cache miss, compiling and saving to cachetriton_bundler_metatime_taken_nsz.Saving compiled graph to FX cache with key: %shitzFX cache hit with key: %sdisabledfx_graph_cache_)r  time_nsr  r  
componentszcache not enabled)r  cache_event_timer  r  r  remote_cache_enabledlocal_cache_enabledartifactc                    > ST  3SS.$ )Nr
  jsonr   encodingr   )r  s   r   <lambda>#_compile_fx_inner.<locals>.<lambda>V  s    -k]; &%r   c                 0   > [         R                  " T 5      $ r   )r  dumps)
cache_infos   r   r  r  Z  s    4::j#9r   metadata_fn
payload_fnz%FX codegen and compilation took %.3fsaten_mm_info_   -?   )r  r  z$Overview info of inductor aten mms: z3{:<30} | {:<20} | {:<20} | {:<20} | {:<20} | {:<20}NameBMNKCountz----------------------------------------------------------------------------------------------------------------------------------ztorchinductor done compiling 	BACKWARDSFORWARDS graph r  )jrt   aot_compilationr   pipeline_max_autotune_gemm torch._inductor.autotune_processr  get_instancewarm_upr   	_inductorasync_compileCompiledTritonKernelscache_clearr   count_callsr   torch._dynamo.utilsr  torch.fx._lazy_graph_moduler  force_recompiler   CompileContextcurrent_compile_idr"   log_instant_eventPT2_COMPILEr-   forwardr  static_inputs_logdebugget_input_idxs_to_checkr   nextiterreversedr  r   r;  r   r   r@   tritonr  	save_argsr8   timerG   r^   allrC   r%   r  fx_graph_cache
_functorchbundled_autograd_cacher)   r   r?  r   rE   r!  r  _is_inductor_staticr;   r  r0   prepare_keyget_remote_cacheload_with_keyrj   begin_compiler  collectset_triton_bundlerU   rV   	Exceptionra   r	   with_traceback__traceback__end_compile_time_taken_ns_fx_graph_cache_key_fx_graph_cache_debug_linesr  _save_graphinstanttry_add_pt2_compilerK   post_compileisEnabledForr   INFOr#   itemsr  r   r   r   endswithr  formatr   )+r   rg  graph_kwargsr  r  pool_instancer  r  r  r  inputs_to_checkstartfx_graph_remote_cachebackends_support_caching	use_cachelocalremoter  inputmb_compiled_graphkey_inforemote_cacher   
start_timer  debug_linestriton_bundler  e	cache_keycompiled_graphmm_table_datar   partsr   
is_batchedbatchmr  krowr  r  s+                                            @@r   r  r  <  s*
    &&H((H+88: 
OO!!77CCE)Q.x 	=@((,]]11DDF
,,"J/*66	
 rzz**'3'>'>?RTV'WDFWX-nPOd4 89:??BUDMRR 
QRTRZRZQ[\R %-%.v}}/G/G%H\"&	
 	
 IIKE<> " 	#
 *"-	
	# 	  
 d
 +++ C&&?*?CC )C $$++BBB 	 %%&
I.		f''	
 ".1HAu5%,,//5<<,,--**,0) 2 37
226	 \\^
%1%=%=L6&"Xz
 ##+ [		6<#/#@#@#BLII560<0J0J"  , 0 0 F'1-!: 		;<""99$,,,%%%
 ''),$:%;G%! )444 "))+!'!33MB ))+
 :m#<#H$,,,II, "- NN#8)DE$:%;G%! &&0$,,,'''IIDE''),$:%;G%! )44437<<>J3N!0)1&	;8A!5@K!= "))+!'!33MB ))+".478K4L
01*;*J*JJ'IIF	R$$! m,555$000''''/$YII19=4=1<G9 ,,,* *4)?J}%Z 	 	""k]+%2	
 	..#')3
u%7Az~~l3t  45(!' %	
  ! : 	##NI|Lk
n II5tyy{U7JK %%">288:JCIIcNE5zA~$$c3S#u%EF ,/u:?388E#2J'sPR@TD'9:Jc%jAo!&rsq!Qxxcr
+$$dE1aE%BC  *1axxcr
+$$dCAq%%@A) ;, 	78AHHS#sG	

 	 CHHJQQSVWXHHY ! 
OO!!77CCEN'&}5;:
F Gj)*	, g %i0  #A|~6EEOO
 ))+&  #A|~6EEOO2 %i0  #A|~6EEOO
 ))+]
 
s   5C1l*"llD/l;g4Ali%A l&A+jE#l4i.h<<iiil
j(.jjlk+8.k&&k++k..ll
lc                  .    \ rS rSr% SrS\S'   SS jrSrg)	_FxCompileStati  r   r  codegen_and_compilec                     SU R                    3$ )Nzcodegen_and_compile: )r  )r  s    r   __repr___FxCompileStat.__repr__  s    &t'?'?&@AAr   r   N)r  r  )r   r   r   r   r  r   r  r   r   r   r   r  r    s      Br   r  c                  p    \ rS rSr% Sr\" \5      rS\S'   \	          S	S j5       r
\S
S j5       rSrg)	FxCompilei  zU
An FxCompile represents a mechanism that can turn a GraphModule into an
OutputCode.
z%dict[type[FxCompile], _FxCompileStat]_compile_statsc                    g r   r   )r  r   rg  rf  rd  s        r   r  FxCompile.codegen_and_compile  s     r   c                8    U R                   R                  5         g r   )r  clear)clss    r   _reset_statsFxCompile._reset_stats  s      "r   r   N
r   rM   rg  r  rf  r  rd  r  r  r>   r  rs  )r   r   r   r   __doc__r   r  r  r   r   r  classmethodr  r   r   r   r   r  r    sr     =H<WN9W
  , '	
 ' 
  # #r   r  c                  <    \ rS rSr\          SS j5       rSrg)_InProcessFxCompilei  c                "  ^^3^4^5^6^7 SU;   a  US   c   eUS   nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      n	UR                  S	S5      n
[        R                  nUR                  S
S5      nUR                  SS5      n[        S5      R	                  5          [
        R                  " 5          [        R                  =nb,  SSK	n[        R                  SU5        UR                  " U5        [        T5      (       a
  [        5         [        S   R!                  5       n["        R$                  " ['        ["        R(                  " 5       S5      5        [+        5       " [,        R.                  SU(       a  SOS SU 35        [0        R2                  " 5       n[4        R6                  R8                  R:                  R=                  UTUSSS9  UR?                  5       m7[A        SS U74S jS9  [        RB                  RE                  TU5        TRF                  nUc  [I        U5      n[K        T5        [M        SSS9   [4        RN                  " 5          [Q        TU5      nSSS5        SSS5        [S        T5        [A        SS U4S jS9  [        RT                  " W5         [W        T5      nU   [Y        TUS9  SSS5        [        RB                  R[                  TU5        [\        RC                  S [_        S!TSSSS"95        TRa                  SSSSS#9m4[A        SS$ U44S% jS9  [        Rb                  Rd                  S:w  a~  [4        Rf                  Rh                  Rk                  TRl                  5      n[o        [4        Rp                  RB                  Rr                  U5      [4        Rp                  RB                  l:        [w        5       nURy                  5       (       a,  [        S&   R{                  5       n[|        R~                  " SUS'9  [        R                  " 5       (       a   [        S([        [        5       5      0S)9  SSS5        [        RT                  " U5         [        U5         [        X5         SnSnSnSnU(       a  [        R                  R                  (       a  [        TS+ S,9u  nn[        U/ UUU	UUUUSU
S-9n[        R                  " U5         [        R                  " / 5         U	(       d   S.5       eUR                  5         UR                  5       u  nnSSS5        SSS5        [        TUUUU	UUUUUU(       a  UR                  OSU(       a  UR                  OSUUU
S/9n[        R                  " 5       nUR                  5         [        R                  " U5         [        R                  " / 5         [        R                  " 5          UR                  " U6   / nUR                  b  [        5       m6UR                   H  n [        U [        5      (       ay  U R                  5       (       ad  [        [        U R                  5       5      5      S:X  a>  UR                  [        U64S0 jU R                  5       R                   5       5      5        M  UR                  S5        M     [        U5        Sn![M        S1SS9   UR                  (       ag  UR                  (       aV  UR                  (       a   eUR                  5       S   R                  n"[        RB                  " S2U"Ra                  SS395        GOUR                  (       Ga`  S4S5KhJin#  UR                  (       d   S.5       eUR                  5       u  n$n%[        RB                  " S6U$R                  5        U%R                  (       a!  [        RB                  " S7U%R                  5        Sn&[        R                  (       a6  UR                  [        R                  5      n&[        RB                  " S8U&5        [M        S9SS9   U#R                  UU$R                  U%R                  U&UR                  / [        R                  UR                  R                  U(       a  UR                  R                  O/ -   5      QS:9n"SSS5        O)UR                  5       n'U'R                  n"[        U'S;S5      n!SSS5        Sm5Sm3[        Rb                  Rd                  S:w  a  [        R                  " [4        Rp                  RB                  R                  5       5      m5[        R                  " [4        Rp                  RB                  R                  5      m3[A        SS< U54S= jS9  [A        SS> U34S? jS9  T3(       a1  [w        5       nURy                  5       (       a  UR                  S@T35        Sn([        R                  [,        R.                  5      (       az  UR                  5       u  n)n*n([        =R                  U)-  sl}        [        =R                  U(-  sl~        [        =R                  U*-  sl        [        GR                  SAU)U*U(SB.5        [        GR                  (       a>  UR                  5       u    n+n([4        Rp                  RB                  GR                  U(5        [4        Rp                  RB                  GR                  UGR                  GR
                  5        U(       Gaz  [        GR                  GR                  (       GaX  [        GR                  (       GdA  [        Rl                  GR                  (       Gd   [4        Rp                  GR                  GR                  " U6 (       a  Sn,TRl                  GR
                   H  n-U-GR                  R                  SCS5      n.U-GR                  SD:X  dP  [        U.[4        GR                  5      (       a0  [4        Rp                  GR                  GR                  U.5      (       d  M  U-GR                  R                  SES5      =n,(       d  M    O   SFn/U,(       a	  U/ SGU, SH3n/OU/ SH3n/U/[        Rl                  l        U(       a  [        GR                  (       d  [        Rl                  GR                  (       dd  G[        T5      n0U0(       aQ  SIU0GR                    3n/U0GR                  R                  SES5      =n,(       a  U/ SGU, SH3n/U/[        Rl                  l        [        R                  (       a  [        W"[        G["        [4        Rf                  GR$                  45      (       d   G['        U"5      5       eG[)        U"UR                  SJ9sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        $ U(       aZ  [        Rl                  GR                  (       d:  SSKKJn1  U1" [        Rl                  GR.                  5      [        Rl                  l        U GR0                  G['        U 5         =GR2                  S4-  sl        [4        Rp                  RB                  GR4                  (       a  [4        Rp                  RB                  GR6                  bp  [        [4        GR8                  GR:                  GR=                  5       5      n2UR                  S5      nUb(  U2[4        Rp                  RB                  GR6                  U'   G[?        W"UTU[        Rl                  GR                  UGRA                  5       [        S   U-
  UUUUUT7T4U!T5T35      sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        $ ! , (       d  f       GNy= f! , (       d  f       GN= f! , (       d  f       GNC= f! [         a    [        R                  S*5         GNf = f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN)= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       O= f SSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        g! , (       d  f       g= f)LzC
Generates the OutputCode from the GraphModule and example_inputs.
r  Nr  r   r  Fr  r  r  r{  r  z/pytorch.wait_counter.actual_codegen_and_compiler   z3Sleeping for %s since sleep_sec_TESTING_ONLY is setr  i  ztorchinductor compiling r+  r,  r-  )save_dirr  c                     SSS.$ )Nfx_graph_runnablestringr  r   r   r   r   r  9_InProcessFxCompile.codegen_and_compile.<locals>.<lambda>  s    / (%r   c                    > T $ r   r   )runnable_graph_strs   r   r  r    s    #5r   r  additional_fake_tensor_propTrb  c                     SSS.$ )Nbefore_post_grad_graphr  r  r   r   r   r   r  r  (  s    4 (%r   c                 &   > T R                  SSSS9$ NFTprint_outputinclude_strideinclude_deviceprint_readabler   s   r   r  r  ,  s    2#4#4!&tD $5 $r   r{  %szAFTER POST GRADr  r  colored)r  r  r  fast_sympy_printc                     SSS.$ )Ninductor_post_grad_graphr  r  r   r   r   r   r  r  N  s     :$,)r   c                    > T $ r   r   )inductor_post_grad_graph_strs   r   r  r  R  s    'Cr   graph_break)	overwritenum_graph_breakspt2_configs)extra_loggingzfailed to log pt2_configsc                   U R                   S:H  =(       am    [        U R                  [        5      =(       aL    U R                  R	                  S5      =(       d*    [        U R
                  R                  SS 5      [        5      $ )Nr
  r  r   )r   r   r   r  r   r   r   rI   )r   s    r   r  r    s_    $''Z:O ;&t{{C8; !KK223CD X)$))--t*DFVW	;r   )r  )
rg  	shape_envr  r  r  r  r{  r  is_const_graphr  z"AOT mode only supports C++ wrapper)rg  r  r  r  r  r  r{  r  r  const_wrapper_codeconst_kernel_codeconst_modulerf  r  c              3  F   >#    U  H  nTR                  U5      v   M     g 7fr   )doprint)r   sps     r   r   :_InProcessFxCompile.codegen_and_compile.<locals>.<genexpr>  s     )X@W1!))A,,@Ws   !zGraphLowering.compile_to_fnzOutput graph module: 
%s)r  rZ   )AotCodeCompilerzOutput wrapper code: 
%szOutput kernel code:
%sz#Serialized Extern Kernel Nodes: 
%szAotCodeCompiler.compile)device_typeadditional_filesrunnerc                     SSS.$ )N*inductor_provenance_tracking_node_mappingsr  r  r   r   r   r   r  r  (  s    (T,21r   c                    > T $ r   r   )r  s   r   r  r  ,  s    /Yr   c                     SSS.$ )N0inductor_provenance_tracking_kernel_stack_tracesr  r  r   r   r   r   r  r  0  s    (Z,21r   c                    > T $ r   r   )inductor_kernel_stack_trace_strs   r   r  r  4  s    /Nr   inductor_provenancezGraph Metrics:
%s)num_bytes_accessednodes_num_elemnode_runtimesr   r1  stack_tracezWgraph with symbolic shapes inputs and config.triton.cudagraph_skip_dynamic_graphs=True.z Found from 
z,disabling cudagraphs due to incompatible op )filenamer  ) check_lowering_disable_cudagraph)r   rt   r.  rQ   guardr   preserve_rng_stater   sleep_sec_TESTING_ONLYrH  r   warningsleepr  r  r#   copysyssetrecursionlimitr  getrecursionlimitr   r   r`  ioStringIOr   _dynamorepro	after_aotsave_graph_reprogetvaluerK   rA  fx_graphr  rs   rd   r%   no_gradr  r   set_fake_modeget_cuda_device_contextrx  fx_graph_transformedpost_grad_graphs_logr(   r  traceprovenance_tracking_levelr   	tracebackget_graph_provenance_jsonr   r7   r3  _pre_grad_graph_id _inductor_post_to_pre_grad_nodesr'   in_progressr  r"   compilation_metric	is_fbcoder   r  r  rT  r  r  r  r  r  rf   set_graph_handlerset_extern_kernel_nodesruncodegen_with_cpp_wrapperr   r\   CachedMetricsHelperfreeze_runtime_assertsr[   graph_contextgraph_outputsrO   r   rh   has_tensor_outputr   rN   
get_strider   r;  
get_layoutr   _check_triton_bf16_supportr  r  r  codegenr   r1   	codecacher  extern_kernel_nodesr  compiler  dictfromkeyswrapper_coder  compile_to_modulecallr   r  r  dump_inductor_provenance_info_inductor_kernel_stack_trace
add_to_setinductor_metrics_logr_  count_bytesr  r  r  r  log_tlparselog_runtime_and_tensor_metalog_collective_schedule	schedulerr  rF  cudagraph_skip_dynamic_graphsr  disable_cudagraphs_reasonr   any_is_symbolicr   r   r   ro   r   r   rM   r  r9   torch._inductor.cudagraph_utilsr  device_node_mappingr  r  RECORD_GRAPH_EXECUTIONGRAPH_COMPILE_IDSr   r;  r<  r:   
get_deltas)8r  r   rg  rf  rd  r  r  r  r  r  r  r  r{  r  	sleep_secrH  inductor_countersfdr  r  cuda_contextprovenance_tracking_jsonmetrics_contextr  r  const_graphr  r  r  r   metrics_helperr   rP  compiled_fn_runnercompiled_fnr  r  kernel_codeserialized_extern_kernel_nodescompiled_moduler  	num_bytesr  r  r  r   meta_valdisablemaybe_incompat_noder  r  r  r  r  r  r  s8    `                                                 @@@@@r   r  '_InProcessFxCompile.codegen_and_compile  sB     |+\0J0VVV ,\ :
+7+;+;<OQS+T(,,]EB"."2"2:t"D(,,]EB'++L%@
**)--neD5t< 	
 JKQQS++-#:::	GI9 

9%)"--#% ( 4 9 9 ; !!#c&;&;&=t"DEN*"-;:> ?!
$ BMM))::B
T ;  "$ 6 GGR0
 I 1.A	$ B-T ]]_ 0^ DI % 6b9
 	 +6r:!/N ",,R@$***)'+'+ $	 02/@/@!&#'#'%)	 0A 0, !!  D <<99Q>**DDRXXN - ;!OO11DD4 OO))J #6"7"..00'/'>'D'D'F$&99"&9I ##%%	A, -s3J3L/M+o ,D 	*3NC-kD%)""%)"$(! 3 3 P P 4B.40H0 #0 ')"+!)$/!)/E%1$/'+#-#K ++K811"5*P,PP{#)'@@B >*,=	 6 9 & $2'% +%+A!- +'94F*00D 4E)//$!,$3)-0 ")!<!<!> ,,.''.--b1(668II~.QSN**6 +,#(#6#6C *3 7 7$'$9$9$;$;$'(=cnn>N(O$PTU$U !/ 5 5$))X@P@W@W)X$X!" !/ 5 5d ; $7 /u5 *.&%5T !>>e.>.>','8'88#8*/--/!*<*?*?K+11 ; + : : : N
 #^^^B#(#4#4  D#4 9>8V8V8X5L++11 ;\=O=O  +00 / 5 5$={?P?P!" >B: 44$)$@$@AVAV$W !? !0 5 5$J$B!"
 ". 9QU" />.E.E$)$0$6$6$/$5$5$B050A0A	6&)-,1,>,>,O,O 4? 1<0H0H0Y0Y57	-.**	6& /F /"	" ", /4.E.E.GO*9*>*>K18 /42.yB BF>6:3||==BEIZZ!OO11OOQFB ;?**!OO11NN;7 )&) (Z )&) (O ;.A.CO.::<< / : :$9$C!"
 %)M+88FFCHCTCTCV@	>=22i?2-->-...@.,1106?2@1> ))).3.?.?.A+1m--II-X OO))AA%//BWBWX #"MMGGG & 6 6 6 ! A A A!OO11AA>R&*$&HHNND'+yy}}UD'AH $= 8'1(ELL'I'I','<'<'L'LX'V'V (.2iimmM4.PP{P % %3 #|&)0	k]"&MG)0	nG<C9
 # & 6 6 6 ! A A A.STV.W+.(TUhUoUoTp&qG.A.F.F.J.J -t/  {   .5I\+b*Q@GAGG= (()' $(<(<=    -  ,	-  
  ,%0e>O>O [ 98 21 /.S ED DC +* .- TS@ "!''*K*K*K = ! ; ; 9 ''T
3GG1LG --DDD!OO11CCO%(!MM88KKM&
 $0#3#3J#?#/ !+ "OO11CCHM +#&99&113 ,/@@"&)$'*4*B7%g 98 21 /.S ED DC +* .- TSX %_ 8 "\r % A $?@Ay ,+N 65 98T" "I ; 988 211 /..S EDD DCC +** .-- TSSso  AE$FAE
"A@	87	A@	9AE
AAA@	E,AAA@-(AE
AD0AD	A&AC<AA72AA%AA7B AC<AC",ACC2AB-4E&ABA<AB	2ABM<AB-EAB-		AC	AC"	AC<$	AD	-	AD06	AE
?	AE$FAB-.	AC7	AC" 	AC<		AD		AD0	AE
$	AE$7
A@@A@	@	
A@@AE
@
A@*@%AA@-AA	AAAAAA	AAAA
AA"AAE
A%
AA4A/AA7A7
ABBAC<B	
ABBABB
AB*B%AB-B-
AB;B7ACB?	AC"C
ACCAC"C	AC<C"
AC0C,AC<C3	AD	C<
AD
DAD	D	AD0D
AD$D AD0D'	AE
D0
AD>D:AE
E	AE$E

AE	EAE$E$
AE2r   Nr  )r   r   r   r   r   r  r   r   r   r   r  r    sK    PP ,P '	P
 'P 
P Pr   r  c                   [         [        R                  :X  a  [        5       nOK[         [        R                  :X  a  SSKJn  U" 5       nO%[         [        R                  :X  a  SSKJ	n  U" 5       n[        (       a,  SSKJn  SSKJn  [        WU5      (       d   S5       eU" U5      n[        (       aA  SSKJn	  SSKJn  [        WU5      (       d   S5       e[#        5       n
[        5       nU	" XU
5      nWR%                  XX#5      $ )	NrZ   )_DebugSerdeFxCompile)_SubprocessFxCompile)_AsyncFxCompile)_OutOfProcessFxCompilez7async is only valid with an out-of-process compile mode)_ProgressiveFxCompilez=progressive is only valid with an out-of-process compile mode)fx_compile_moder   r   r  r   compile_fx_extr-  r   compile_fx_subprocr.  fx_compile_asynccompile_fx_asyncr/  r0  r   fx_compile_progressiver1  r   r  )r   rg  rf  rd  schemer-  r.  r/  r0  r1  progression_configsfast_schemes               r   r  r    s     -...$&	M33	38%'	M44	4<%'5: &"899 	
E	
9 !(;: &"899 	
K	
9 78 *+ '{<OP %%b/XXr   c                   / n[        U 5       H  u  p4[        U[        R                  5      (       d  M&  [	        UR
                  R                  5      (       d  ML  [        5          X1;   a  [        U5      (       a   SSS5        Mw  [        U5      (       d   SSS5        M   SSS5        UR                  U5        M     U$ ! , (       d  f       N%= f)z
This function runs at compile time, and generates a list of indices for which we
might need to do a copy to preserve alignment requirements.
N)r?  r   r   r   rE   r!  r  rp   rH   rF   r   )inputsr  ids_to_checkr  rm  s        r   rB  rB    s     Lf%%..ell''((02 %*;E*B*B	 32
 /u55 32
 6 3 	A) &,  32s   $C C  
C	r   )r   placeholdersmutated_input_idxsc                 ^ ^^^ SSK Jn	  [        R                  R                  (       aC  [
        R                  " U	UUUUUUU[        R                  R                  R                  5       S9	mO[        mS mSUUU U4S jjn
U
$ )Nr   )cudagraphify_impl)device_indexstack_tracesr  r{  r   r>  r?  r  c                   > Tc(  [         R                  " 5          T" TU T5      mS S S 5        T" U 5      $ ! , (       d  f       N= fr   )r   r  )
new_inputsr#  cudagraphify_fnmodelr  s    r   r  cudagraphify.<locals>.runB  s=    002-eZARS 3:&& 32s	   5
A)rE  r  r  r   )torch._inductor.cudagraph_treesrA  r   rF  cudagraph_trees	functoolspartialr   r   r;  r<  )rG  r  rB  rC  r  r{  r   r>  r?  new_cudagraphify_implr  r#  rF  s   ``         @@r   cudagraphifyrN     sr    
 }}$$#++!%%#%%1}}33FFH

 ,K' ' Jr   c                    [         R                  " U R                  5       U R                  5       U R                  U R
                  S9$ )z)
Copy and input while preserving strides
)r"  r!  )r   empty_stridedsizer   r"  r!  )r   s    r   static_inputrR  L  s/     qvvx177188TTr   c                R    [        X5      n [        X5      nU R                  U5        g)z=Index into expanded dimensions of both dst and src then copy_N)r=   copy_)dstsrcexpanded_dimss      r   index_expanded_dims_and_copy_rX  S  s#     c
1C
c
1CIIcNr   c                  ^^	^
^^^ [        UT5      n[        [        UT5      5      m[        X5        [	        U[
        5      (       d   e[        U5       VVs/ s H  u  pEUT;  a  [        U5      O/ PM     snnm[        U5       VVs/ s HG  u  pE[	        U[        R                  5      (       d  UO UT;  a  [        U5      OUR                  5       PMI     snnm[        [        UT5      5       H@  u  nu  pV[	        U[        R                  5      (       d  M)  UT;  d  M1  [        TU   XV5        MB     [        R                  R                  5         [        R                  R!                  5       nUR#                  [        R                  R%                  5       5        [        R                  R'                  U5         U " [        T5      5        SSS5        UR                  5         [        R                  R%                  5       R#                  U5        [        R                  R                  5         [        R                  R)                  5       m
[        R                  R+                  T
USS9   U " [        T5      5      mSSS5        [	        T[
        [,        45      (       d  T4m[.        R0                  (       a  SU
UUUU4S jjnO8[3        [5        T5      5       Vs/ s H  oDT;  d  M
  UPM     snm	SU	U
UUU4S jjn[7        X[        5       5      $ s  snnf s  snnf ! , (       d  f       GNY= f! , (       d  f       N= fs  snf )zI
Assumes inputs[static_input_idxs[i]] are always the same memory address
Nthread_local)streamcapture_error_modec                  > [        T5      [        U 5      :X  d   e[        [        TU T5      5       H  u  nu  p#n[        U[        R
                  5      (       d  M*  [        U[        R
                  5      (       d   eUT;   a&  UR                  5       UR                  5       :X  d   eMw  [        X#U5        M     U R                  5         TR                  5         T	$ r   )
r   r?  zipr   r   r   data_ptrrX  r  replay)
rE  r   rU  rV  rW  r   inps_expanded_dimsr  static_inputsstatic_outputss
        r   r  cudagraphify_impl.<locals>.run  s    }%Z8882;M:/AB3..c "#u||44!#u||4444++<<>S\\^;;;
 2#MJ3 LLN!!r   c                   > T H<  nTU   nX   n[        U[        R                  5      (       d   e[        TU   X25        M>     U R	                  5         TR                  5         T$ r   )r   r   r   rX  r  r`  )	rE  r   rW  rV  copy_indicesr   ra  rb  rc  s	       r   r  rd    sa    # 23 7 o!#u||4444-mC.@#U	 $
 LLN!!r   )rE  list[InputType]r   Callable[[list[InputType]], Any])rB  rR   rr   rm   r   r   r?  r<   r   r   rR  detachr^  rX  r   synchronizeStreamwait_streamcurrent_streamr[  	CUDAGraphr   r;  r   size_assertsr   r   rk   )rG  r<  r  check_input_idxsr   r   rW  r[  r  rf  r   ra  rb  rc  s     `      @@@@@r   rA  rA  ^  s    /v7HI)3#F,=>* 64fd####  ''FC !$+< <!"D'  '	 (FC a..  ++ a	 (	M $-S9K-L#Maa&&36G+G)-*<aO $N
 
JJZZ F
uzz0023			6	"d=!" 
#
	JJ++F3	JJ JJ  "E			%>		RtM23 
SntUm44(*	" 	", !]!34
4CCT8TC4
		" 		" (z|LL]	* 
#	" 
S	R8
s1   L	AL<L .L2	M+M 
L/2
M c                   [        U [        5      (       d   U 5       e[        U 5        [        R                  " U=(       d    0 5      nUR                  SS5      (       d  [        R                  (       d  SUS'   UR                  S[        R                  R                  5      nU(       a  UR                  S5      (       a   S5       eO0 UES[        U R                  5      0EnSS	KJn  U" U5      nUR                  S
S 5      nU R                   R                  SS 5      n["        R$                  R'                  U5      n[(        R*                  " S5         ["        R$                  R-                  U5         [/        SSSS9   [1        5          [3        U U[4        R6                  " UUS9US9n	[        U	[8        5      (       d   eU	R:                  sS S S 5        sS S S 5        sS S S 5        sS S S 5        $ ! , (       d  f       O= f S S S 5        O! , (       d  f       O= fS S S 5        O! , (       d  f       O= fS S S 5        g ! , (       d  f       g = f)Nr  FTr  zaot_inductor.output_pathz.pt2a
  The output path for aot_compile should not have an extension with .pt2 this is for specifying the output path for the .so in AOTInductor. If you would like to package the AOTInductor generated files into a pt2, please call `torch._inductor.aoti_compile_and_package`.rZ   )maybe_aoti_standalone_configr  dynamo_compile_idcompile_fx_aot)rb  reset_event_log_on_exit)r  )inner_compiler  )r   rM   r*   r  deepcopyr   r   r  r  output_pathrb  r/   coder   rr  r   r   r   r   r;  rt   set_aot_compilationcompile_contextr!   r'   
compile_fxrK  rL  r9   r  )
model_example_inputs_rv  r  rx  rr  r  saved_compile_idsaved_compile_contextcompiled_artifactss
             r   rt  rt    s    fk**2F2* &f- &*]]>3GR%HN|U33v7H7H(,}% $$"F$7$7$C$CK ''// 	
R	
//

&	&++(>

 41.AN+//0H$O{{':DA!MM889IJ	d#%%&;<"&$(	

 	'#++'= *
 ,l;;;;!** 		
 	
 	=< 	$# 		
 	
 	
 	=<< 	$##sa    I 3H&?H
AG1		H	H&	I 1
G?;H	H&
HH&	I &
H4	0I  
Ic                  ^^^^ SSK JnJn	  [        U 5      n [        R
                  " U SS9n
U
(       a  [        XS5        U" U 5        U	" UU U5      u  nmT Vs/ s H  oU   PM	     nn[        U5      nUR                  R                  Gt pUR                  S   n[        U5       VVs/ s H3  u  nn[        U[        R                  R                  5      (       d  M1  UPM5     snnUR                   S'   / n[        R"                  R$                  R'                  5       nS/mSmUGbA  UR(                  c   eUR(                  n[+        S[-        U5      S-
  5      m[.        [0           " 5       nUR2                  nUc   eSn[-        U5      S:  a  / m[5        [-        U5      5       HN  nUT;  a   S UU'   US:  a  UU   UUS-
     :X  a  US-  nOUR7                  UU   5        TR9                  U5        MP     UR:                  c   e[5        [-        UR:                  5      5       H  nUU;  d  M  S UR:                  U'   M     UR<                  (       a  UR<                  R>                  n[@        RB                  RE                  USS5         U" UUUUUSUU
S9mS S S 5        [F        RH                  (       a  T$ S
UUUU4S	 jjnSUl%        U$ s  snf s  snnf ! , (       d  f       ND= f)Nr   )%convert_conv_weights_to_channels_lastfreezeTr  r   rZ   r  )r  r  r  r{  r  r  c           
        > T Vs/ s H  nXT[        UT5         -
     PM     nnU R                  5         T" U5      $ s  snf r   )minr  )r   r  args_newmax_offset_idxoptimized_functionpreserved_arg_indicesunwrapped_args_offsetss      r   wrapper%fw_compiler_freezing.<locals>.wrapper_  sU     +
* +C>,BCCD* 	 
 	

!(++
s   >)r   zlist[object]r  zSequence[torch.Tensor])&torch._inductor.freezingr  r  rn  rf   decide_layout_optr  r$   r   r  r   r?  r   r   r   r   r   r   r   r   params_flat_unwrap_subclassesr  r   rR   r  params_unwrapped_to_flat_indexr   r%  r   params_flatr   r   r   r  r  rt   r.  _boxed_call)aot_autograd_modelaot_example_inputsdynamo_modelnum_example_inputsrv  r  r  forward_devicer  r  r  	opt_modelindr  r  model_outputs_nodemodel_outputsr   r  r  tracing_contextparams_flat_unwrappreserved_indices_params_flatunwrapped_idxscurrent_offsetr  r  r  r  r  r  s                              @@@@r   fw_compiler_freezingr    s    W 77IJ001CRVWJ+F-.@A'-($I$ >SS=RcS1=RS !34I '__22Q&++A.M#M2;2QjEHHMM6R2;67 $&mm22::<OSN"<<HHH,JJQ$6 7! ;<(23(9%(GG)))!"Q&%'"s-./A--(,"1%q5^A..Q2GG"a'N-11.2CD")).9 0 **666s?6678A5515++A. 9 && / ; ; P P			9&=t	D*/!'5!	
 
E 	!!, , GNQ T;L 
E	Ds   K
)0KK	K
K#c                     [         R                  R                  (       a  [        [	        S5      5        [         R                  R
                  b  [         R                  R
                  O	[        5       SSSS.$ )Nzcpp wrapper enabledFT)ztriton.autotune_at_compile_timeztriton.autotune_cublasLtztriton.cudagraphsztriton.store_cubin)r   rF  r  r4   r3   autotune_at_compile_timerY   r   r   r   get_cpp_wrapper_configr  l  sY    }}+'(=>	
 }}55A MM22$)""
 
r   c                R   [         R                  R                  5       (       d  [        R                  " 5       $ [        S [        U 5       5       5      n[        U5      S:X  a1  [         R                  R                  [        [        U5      5      5      $ [        R                  " 5       $ )zP
Returns a cuda device context manager if there is a single device in the graph
c              3  H   #    U  H  oR                   S :X  d  M  Uv   M     g7f)r   N)r  r  s     r   r   *get_cuda_device_context.<locals>.<genexpr>  s      806KK64I0s   "	"rZ   )r   r   r   r  r  rR   rC   r   r!  rC  rD  )r   cuda_devicess     r   r  r    s     ::""$$%%''-7 8,R08 .L |! 	

$tL123 ##%r   c                N   [        U 5      nU   [        U SS9n S S S 5        UR                  SS 5      n[        R                  c/  [
        R                  " SSS9   [        U U4SUS.UD6sS S S 5        $ [        [        R                  [        5      (       d   e[
        R                  " [        R                  R                  R                  SS9   [        R                  " U U4SUS.UD6sS S S 5        $ ! , (       d  f       N= f! , (       d  f       g = f! , (       d  f       g = f)NTrr  static_lifetime_input_indicesr   r  r  )compilerr  )r  rn  r   r   custom_partitioner_fnr   r%   r   r   r6   	__class__r   )r   joint_inputsr   r  r  s        r   partition_fnr    s$   
 +2.L	 +2DI	 
 :@':! ##+&&1
 7 $.K	
 
 
 &668KLLLL&&((22;;"&
 // $.K	
 	
 
/ 

 

 
s#   C4DD4
D
D
D$c                h    [        U 5      n[        R                  " UR                  6 n[	        U5      $ r   )rq   r@  arg_tree_leavesr   r   )rG  r  r  s      r   get_num_model_outputsr    s/    $U+**,>,C,CDM}r   )frozenc                  4    \ rS rSr% S\S'   S\S'   S\S'   Srg	)
CompilerConfigExtrai  r@   r  r  r  r2   r  r   Nr   r   r   r   r  r    s    M$$r   r  c                    [        U R                  R                  5      n[        [        5      n[        S 5      n[        UUUS9$ )N)r  r  r  )r@   rF  r  rC  _graph_counterr2   r  )r   r  r  r  s       r   create_compiler_config_extrar    sF    
 6==334J N#H &d+N% r   c           
       ^  U(       a+  [        SS U 4S jS9  [        T 5      m [        SS U 4S jS9  [        R                  R                  R                  U[        U5      5      n[        T 5      n[        R                  (       a  [        R                  " UR                  6 n	[        U	5      n
[        R                  R                  R                  5       nUb/  UR                   (       a  U(       d  UR                   R"                  nOSnX*::  d   eX-   nX::  d   e[%        X5       Vs/ s H2  n['        X   [        R(                  R*                  5      (       d  M0  UPM4     snUR,                  S'   O/ UR,                  S'   [/        T 5        U" T U[1        U5      UR2                  UR4                  UUR6                  S	9$ s  snf )
a  
Compile the forward graph of the given graph module.

Args:
    gm: The graph module to compile.
    example_inputs: The example inputs to use for compilation.
    num_orig_model_outputs: The number of model outputs from the original dynamo graph.
    num_example_inputs: The number of example inputs from the original dynamo graph.
    compiler_config_extra: Extra configuration for the compiler.
    inner_compile: The inner compile function to use.
    is_inference: Whether this is an inference graph.
r  c                     SSS.$ )Nbefore_joint_graphr  r  r   r   r   r   r  $compile_fx_forward.<locals>.<lambda>  s    ,$!r   c                 &   > T R                  SSSS9$ r  r  r  s   r   r  r        r00"4  1  r   r  c                     SSS.$ )Nafter_joint_graphr  r  r   r   r   r   r  r  	  s    +$!r   c                 &   > T R                  SSSS9$ r  r  r  s   r   r  r  
	  r  r   r   r   )r  r  r  r{  r  )rK   rn  r   r3  r   num_fw_fixed_argumentsr   rq   r   keep_output_strider@  r  r   r   r   r   r   num_mutated_inp_runtime_indicesr   r   r   r   r   r   r   r  r  r  )r   rg  num_orig_model_outputsr  compiler_config_extrarv  r{  r   r  r  num_model_outputsr   original_output_start_indexorig_output_end_idxr   s   `              r   compile_fx_forwardr    s   , 		
 +2.		
 OO!!88C/E %R  ..0B0G0GH.--..6687#6#6|##CC ( +,'%::: :R #777 8N?
N-,ehhmm< N?
 :; ?A :;
 /r2
/6(33&//!#8#G#G ?
s   */GGc                   SSK Jn  U   [        U 5      n[        R                  (       av  [
        R                  " UR                  6 n[        U5       VVs/ s H2  u  px[        U[        R                  R                  5      (       d  M0  UPM4     snnUR                  S'   O/ UR                  S'   [        U 5      n	[        R                  (       a  [        R                   " [#        5       5      O[$        R&                  " 5          U" U U[)        [+        U	5      5      UR,                  SUR.                  UR0                  S9sSSS5        sSSS5        $ s  snnf ! , (       d  f       O= f SSS5        g! , (       d  f       g= f)a  
Compile the backward graph of the given graph module.

Args:
    gm: The graph module to compile.
    example_inputs: The example inputs to use for compilation.
    compiler_config_extra: Extra configuration for the compiler.
    inner_compile: The inner compile function to use.
r   )compile_lockr   T)r  r  r  r  r  N)torch._dynamo.convert_framer  rq   r   bw_outputs_user_visibler@  r  r   r?  r   r   r   r   r   rA   r  r  r  r  r  r   r   r  r  r  )
r   rg  r  rv  r  r  r  r   r  r   s
             r   compile_fx_backwardr  M	  s4    9	(_))"224F4K4KLM (6C6FCa/ 6C##$>? CE##$>?r" !! LL/12'')* !"&uU|"40;; .77+@+O+O	* * 
C* * * 
s=   AE2/EEA4E2<E>	E2E2
E%	!E22
F c                  ^  [        SS U 4S jS9  [        R                  S[        ST SSSS95        [	        T R
                  5      [        R                  R                  l        [        R                  R                  S	:X  ak  T R
                  R                   HQ  nUR                  (       d  M  UR                  [        R                  R                  R                  UR                  '   MS     [!        T U5      m [        SS
 U 4S jS9  T $ )Nr  c                     SSS.$ )Nbefore_pre_grad_graphr  r  r   r   r   r   r  %run_pre_grad_passes.<locals>.<lambda>	  s    + 
r   c                 X   > T R                  SSSS9S[        T R                  5       3-   $ NFTr  z

 # graph id: r  idr   r}  s   r   r  r  	  7    600tD 1 
 b./
01r   r  r  zBEFORE PRE GRADTr  rZ   c                     SSS.$ )Nafter_pre_grad_graphr  r  r   r   r   r   r  r  	  s    * 
r   c                 X   > T R                  SSSS9S[        T R                  5       3-   $ r  r  r  s   r   r  r  	  r  r   )rK   pre_grad_graphs_logrA  r(   r  r   r   r3  r  r   r  r  r  r  #_inductor_pre_grad_node_stack_tracer   r_  )r}  r~  r   s   `  r   run_pre_grad_passesr  {	  s    
 
1
 	
	 02&,,/?EOO,||--2LL&&D$$ %%II$))T ' (@F
1
 Mr   c                >   SSK Jn  UR                  SS5      (       a  U $ U(       aG  [        R                  " U5         [        U U[        R                  " U5      " U5      UUS9sSSS5        $ [        S U 5       5      (       a2  [        R                  R                  R                  R                  5         [        R                  (       d  [        R                  (       a  SSKJn  [        R                  n[        R                  n	[        R                  " [!        5       5         ["        R$                  " U5         ['        U [(        5      (       a  [+        X5      OUn
[-        U
5      nU" X
0 U5       u  nnnnn[/        UU[0        R2                  " UUU	S	9UUS9sSSS5        sSSS5        sSSS5        $ [/        U UUUU5      $ ! , (       d  f       GNg= f! , (       d  f       O= f SSS5        O! , (       d  f       O= fSSS5        MW  ! , (       d  f       Ne= f)
a  
Main entry point for compiling given FX graph.  Despite the fact that this
lives in :mod:`torch._inductor`, this function is responsible for calling
into AOT Autograd (and we will eventually get a callback to
``inner_compile`` to perform actual compilation.  In other words, this
function orchestrates end-to-end compilation for the inductor backend when
you use :func:`torch.compile`.

NB: This function TAKES OWNERSHIP of the input ``model_`` and can potentially
mutate it!  Make a copy if you need to preserve the original GraphModule.
r   )CompilerBisectorr  pre_grad_graphrv  decompositionsignore_shape_envNc              3     #    U  H?  n[        U[        R                  5      =(       a    UR                  R                  S ;   v   MA     g7f))r   xpuN)r   r   r   r!  r  )r   rt  s     r   r   compile_fx.<locals>.<genexpr>	  s7       A 	1ell#H(HH s   AA	)_fakify_script_objects)r  r  )!torch._inductor.compiler_bisectorr  disable_subsystemr   r  r|  r  r   r3  r4  AsyncCompilewakeupr  r  torch._export.non_strict_utilsr  r  rt   set_real_inputsr   rM    _extract_inputs_from_exported_gmr$   _maybe_wrap_and_compile_fx_mainrK  rL  )r}  r~  rv  r  r  r  r  r  cpp_wrapper_configfx_wrapper_configinputs_r  patched_mod	fake_argsr  s                  r   r|  r|  	  s   , C))*6FGG\\.)$ll>:=I-!1 *)      	%%2299;V..I#//"-- LL/12o. fk22 1I$ 
 )1I'YG L6"+"3"3%$6#4#
 $2%5
 HG /. 328 + i *)B HGG /.. 322sM   'G%H<8G34'G	G3$	H
G
G&"G3*	H3
H	=H
Hc           
        U R                   R                   Vs/ s H/  o"R                  S:X  d  M  UR                  R	                  S5      PM1     nn[
        R                  (       d3  U Vs/ s H&  n[        U[        R                  5      (       a  UOS PM(     nn[        S U 5       5      (       a  [        [        5       X15       H  u  pVnUc  M  [        U[        R                  5      (       d  M,  [        U[        R                  5      (       d   eUR                  UR                  :w  d  Mi  [        SU SUR                   SUR                   S35      e   U$ U$ s  snf s  snf )Nr1  r   c              3  (   #    U  H  oS Lv   M
     g 7fr   r   )r   vs     r   r   3_extract_inputs_from_exported_gm.<locals>.<genexpr>
  s     
.+QD=+s   zBDevice mismatch between fake input and example input at position #z: z vs zx. If the model was exported via torch.export(), make sure torch.export() and torch.aot_compile() run on the same device.)r   r  r   r   r   r   r  r   r   r   r  r^  r
   r!  
ValueError)r   r~  r   fake_inputsinpr   fir  s           r   r  r  
  s8    *,)777m;S		e   
 GR
FQs:c5<<00Cd:k 	 
 
.+
...eg{DJCQ~*R">">!!U\\222299($\]`\aac99+T!(( 4cc 	 E 3
s   EE*-E!c                h   [         R                  " [        UUUS9n[        U 5      (       d  [	        XU5      $ [        U [        5      (       a5  [        U R                  R                  [        5      (       a  [        XU5      $ [        S U 5       5      (       a  [        XU5      $ [        U UUUU5      $ )z
Part of compile_fx, called after patching configs.

Ultimately we want to call _compile_fx_main, where the actual work happens.
But under various conditions, various forms of wrapping might be needed
around _compile_fx_main.
r  c              3  X   #    U  H   n[        U[        [        [        45      v   M"     g 7fr   )r   r   r;  r  r   s     r   r   2_maybe_wrap_and_compile_fx_main.<locals>.<genexpr>E
  s!     
G!:a$t,--s   (*)rK  rL  r  graph_returns_tuplemake_graph_return_tupler   rM   r   _codegenrX   handle_dynamo_export_graphr  r&   _compile_fx_main)r}  r~  rv  r  r  
compile_gms         r   r  r  &
  s      ""'#%)	J v&&&v
KK&+&&:~, , *&:NN

G
GGG $FZHH  r   c                  ^ ^^^ [        [        R                  5         [        5          [        R
                  R                  R                  [        R                  R                  S:H  5         [        R                  R                  R                  5          [        T [        5      (       a  [!        T U5      m [        R"                  (       a   e[%        U5      m['        [        5      mUb  UO	[)        5       n        SUUU U4S jjn[*        R,                  " USS9n[/        [0        U5      n[        R2                  (       aW  [        R4                  " 5       (       d=  [*        R,                  " [6        T TTTR8                  TR:                  TR<                  S9nO%[*        R,                  " USS9n[/        [0        U5      n[?        SS	9      SUU4S
 jj5       n[/        [0        U5      n[A        U5      =(       d    [        RB                  RE                  SS9n	[        RF                  RH                  RK                  5       =(       d    [        RF                  RI                  U	5      n
[L        RN                  (       Ga  [        RP                  (       Gd  SSK)J*n  U" 5         [V        RX                  " S[        RZ                  S9   []        T USUS9u  pSSK/J0n  U" U5      n	URb                  Rd                   GH  nURf                  S:X  d  M  SURh                  ;  d  M(  [k        URl                  5      " U5      n[        U[        Rn                  5      (       a$  U	c   eU	Rq                  USS9URh                  S'   M  [        U[        Rr                  5      (       d  [u        [w        U5      5      (       a9  [        Rx                  Rz                  R}                  U	U5      URh                  S'   M  [        U[~        5      (       d  GM  UURh                  S'   GM!     SSS5        [        T WW5      nST Rh                  ;   a  T Rh                  S   URh                  S'   ST Rh                  ;   a  T Rh                  S   URh                  S'   [        R                  R                  5       nU(       a  [        R                  R                  O[        R                  n[L        R                  " U	5         [        R                  " 5          U" 5          U" UU5      sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        $ [L        R                  " U	5         [        RF                  R                  U
5         [        R                  " 5          [V        RX                  " S[        RZ                  S9    [        UUUU[        STR8                  TR<                  US9	" T U5      sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        sSSS5        $ ! , (       d  f       GN*= f! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        GM2  ! , (       d  f       GNA= f! [         a  nUR                  5       SeSnAff = f! , (       d  f       O= f SSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        O! , (       d  f       O= fSSS5        g! , (       d  f       g= f)a)  
Main part of compile_fx, called after wrapping is done.

Roughly speaking, here the steps will be:
(1) apply pre-grad passes
(2) create `fw_compiler` and `bw_compiler` functions out of `inner_compile`
(3) call aot_autograd, which:
- (3a) creates a joint graph with `decompositions`,
- (3b) partitions it with `partition_fn` into fw and bw graphs (applying joint-graph passes),
- (3c) calls `fw_compiler` and `bw_compiler` on those graphs (applying post-grad passes)
- (3d) finally, assembles the fw and bw compiled functions back together and returns.
rZ   Nc                   > [         R                  " S5         [        T[        5      (       a  [	        T5      nO[	        U 5      n[        U UUTTTUS9sS S S 5        $ ! , (       d  f       g = f)Nz$compile_fx.<locals>.fw_compiler_base)r  r  r  rv  r{  )r   r%   r   rM   r  r  )r   rg  r{  r  r  rv  r}  r  s       r   fw_compiler_base*_compile_fx_main.<locals>.fw_compiler_base
  sa    
 **+QRfk22-B6-J*-B2-F*)"+A'9*?"/!- SRRs   ;A
A+Fr  )r  r  rv  r  r  r  Tbackward)r  c                ~   > [         R                  " S5         [        U UTTS9sS S S 5        $ ! , (       d  f       g = f)Nzcompile_fx.<locals>.bw_compiler)r  rv  )r   r%   r  )r   rg  r  rv  s     r   bw_compiler%_compile_fx_main.<locals>.bw_compiler
  s5    
 ))*KL*"*?"/	 MLLs   .
<r  )is_valid_aoti_model_name)unlift_effect_tokensselective_decompose)trace_jointr  r   )_detect_fake_mode_from_gmr
  r   )static_shapes dynamo_flat_name_to_original_fqnrs  )	fw_compilerr  inference_compilerr  r  keep_inference_input_mutationsr  r  r  )r   rM   rg  r  r{  r   r  r>   )r   rM   rg  r  r  r>   )NrW   r  r  r   r   r   r  preserve_node_metar   r  r  r3  rA  reset_provenance_globalsr   rM   r  _raise_error_for_testingr   r  r`   rK  rL  r.   r>   freezingis_grad_enabledr  r  r  r  rL   r$   r  r  r   r   r   rt   r.  enable_autograd_for_aotr   r  functorch_configr  r  r+   torch._export.utilsr  r   r  r   r   r   r   r   from_tensorScriptObjectrJ   r  _libraryfake_class_registrymaybe_to_fake_objrI   rR  _C_is_any_autocast_enabled_DisableAutocastr  r  r  r   _disabletracingrT   r  rU   remove_dynamo_frames)r}  r~  rv  r  r  r  r  r  r  r  r  r  r   rC  r  r   r   rQ  disable_ampr   rt  r  r  s   ` `                  @@r   r  r  T
  s   ( 	}BBC "--LL22a7	
 	668 fk**(AF2222 1 <V D -8N>Q>S 			/	 	 		 	* .UC 	 6j+N??5#8#8#:#:5>5F5F$##5+0;;.774CC6 "+!2!23CRV!W!@." 
'*	=		-@		 
>	 6j+N$
 J--D-I 	 MM((002 7}}++I6 	
 V%C%C%C7$&!''%)$*$>$> '8# %#1	'# J5b9	 HHNNDww*,dii1G!+DKK!8!<%fell;;#,#88#8/8/D/D &d 0E 0DIIe, (0B0BCC~ LH H !& B B T T$-v!" !IIe,
 (0@AA/5DIIe,# +)N (ODK1V[[@GM{{6H  !CD #fkk18>DW8X  !45  ((;;=K-8))j>T>T  +->-G-G-I79)+G LU9-I-I++i 	98	
 	
 	#" 	DCz OOI&MM!!/2&&(""%)$*$>$>
9# + +'9#1!-374??/D/S/S%5
 /
+  )( 32 '&q 	98	
 	
 	#" 	DC~ t LU9-I-I-I++++0 $ 9 ,,.D89#   )(( 322 '&&q 	988	
 	
 	
 	#"" 	DCCs  ]3A].)\?H/\%	AX?X?!C"X?X?C\%	.ZY+	Y	Y+	Z'	\%	0	\?9	]	]3\%	+ \[1!$[Z</Z6	[?	[1	\	\%		\?#	],	]3?
Y	\%	
YY+"	Z+
Y95Z<\%	
Z\%	
Z9#Z44Z99Z<<
[
[	[1
[%![1(	\1
[?;\	\%	
\\%		\?%
\3/\?6	]?
]	]	]3
]'	#]33
^c                   [        U [        5      (       d  g[        U 5      R                  u  n[        U[        [
        45      (       a  g[        U[        R                  R                  R                  5      (       a~  [        UR                  S5      (       ac  [        UR                  R                  R                  5      S:  a6  [        S UR                  R                  R                   5       5      (       a  gg)z"True if a FX graph returns a tupleT_schemarZ   c              3  R   #    U  H  n[        UR                  5      S :H  v   M     g7f)r   N)r  r  )r   rets     r   r   &graph_returns_tuple.<locals>.<genexpr>*  s     O5NcCHH)5Ns   %'F)r   rM   rq   r   r   r;  r   r   r   r   r  r   r   r0  returnsrI  )r   rvs     r   r  r    s    b+&&O  ER"tUm$$2uxx}}))**BIIy))		!!))*Q.ORYY5F5F5N5NOOO r   c                  ^^ [        U 5      nUR                  u  n[        R                  " U5      u  nmU R                  R                  U5         U R                  R                  U5        SSS5        U R                  R                  U5        [        U 5      (       d   eU" X5      m[        R                  " T5      SUU4S jj5       nU$ ! , (       d  f       Ng= f)zu
Mutate gm so it returns a tuple.  This is only needed for graphs
not created by torchdynamo that return non-tuples.
Nc                 >   > [         R                  " T" U 0 UD6T5      $ r   )r@  tree_unflatten)r   r   r#  specs     r   r  (make_graph_return_tuple.<locals>.wrapperD  s     $$[$%A&%A4HHr   )r   r   r   r   r  r   )rq   r   r@  tree_flattenr   inserting_beforer   r  r  rK  wraps)r   r<  r  r   r5  r  r#  r9  s         @@r   r  r  1  s     r?DIIER""2&HB		"	"4	(
 
)HHr""""R(K__[!I "I N 
)	(s   C
Cc                4  ^^ U R                   R                  m[        R                  R                   R	                  5       U R                   l        U R                  5         U" U TR                  " U6 5      m[        R                  " T5      SUU4S jj5       nU$ )z
`torch._dynamo.export` embeds pytrees in the FX graph codegen object,
convert that to a normal FX graph so inductor can compile it.
c                 F   > TR                  T" TR                  " U 6 6 5      $ r   )process_outputsprocess_inputs)r   r   r#  s    r   r  +handle_dynamo_export_graph.<locals>.wrapperZ  s$    &&{G4J4JD4Q'RSSr   )r   r   r  r   )	r   r  r   r   CodeGenr  rA  rK  r=  )r   r<  r  r  r   r#  s       @@r   r  r  K  su     hhG..0BHHLLNR!7!7!@AK__[!T "T Nr   c                   SS jn[         R                  " U R                  R                  5       U R                  5       H  n[        U[        5      (       d  M  [        U5      nU(       a2  [        U5      (       a"  UR                  5       [        R                  :w  a  M`  [        U5      nUR                  SS9(       a    g U" UR                  5       5        M     g )Nc                    SSK Jn  U c   e[        U R                  5      nUR	                  U 5      n[
        R                  " UR                   S35        U" S5      e)Nr   )rV   z9 does not support bfloat16 compilation natively, skippingzBF16 is not supported)torch._dynamo.excrV   r   r  get_device_propertiesr  r  r   )r!  rV   device_interfacedevice_propss       r   warn_and_skip1_check_triton_bf16_support.<locals>.warn_and_skipb  s\    /!!!3FKK@'==fE  !!Z[	
 /00r   F)including_emulation)r!  zOptional[torch.device]r  r   )	itertoolschaingraph_inputsrB  r  r   rh   rg   rE   	get_dtyper   bfloat16r   is_bf16_supported
get_device)r   rJ  r   r  rH  s        r   r  r  a  s    
1  2 2 9 9 ;U=P=PQ$''%d++&&~~5>>1 4K@--%-Hdoo'( Rr   )optionsc                  SSK Jn  U" U 5      (       d   S5       eSnSn[        U R                  R                  [
        R                  R                  R                  5      (       a  U R                  R                  n[
        R                  R                  R                  5       U R                  l        U R                  5         UR                  R                  b  UR                  R                  nUR                  R                  b  UR                  R                  nO:[        U S5      (       a  U R                  n[        U S5      (       a  U R                  nUb  [         R"                  " U5      OSnUb  [         R"                  " U5      OSn	[         R$                  " X=(       d    0 45      u  p['        S U
 5       5      (       a  S	S
KJnJn  U" UR.                  S5      eU
 Vs/ s H,  n[        US   [
        R0                  5      (       a  US   OSPM.     nnUb  X:w  a  [3        SU SU 35      eUc  UU	S.O0 UEUU	S.EnX4$ s  snf )z
Flatten the inputs to the graph module and return the flat inputs and options.
Add "aot_inductor.serialized_in_spec" and "aot_inductor.serialized_out_spec" to the options.
rZ   )r  zGraph output must be a tuple(). This is so that we can avoid pytree processing of the outputs. Please change the module to have tuple outputs.N_in_spec	_out_spec c              3  \   #    U  H"  n[        US    [        R                  5      v   M$     g7f)rZ   N)r   r   r$  r   s     r   r   '_aoti_flatten_inputs.<locals>.<genexpr>  s&     
M9LA:adE..//9Ls   *,r   )	UserErrorUserErrorTypezTorchBind objects found in inputs. TorchBind object inputs are not supported in AOTInductor. TorchBind objects can only be attributes.z>Trying to flatten user inputs with exported input tree spec: 
z-
but actually got inputs with tree spec of: 
)zaot_inductor.serialized_in_specz aot_inductor.serialized_out_spec)r|  r  r   r   r  r   r   rX   rC  r  pytree_infoin_specout_specr  rV  rW  r@  treespec_dumpstree_flatten_with_pathr  rF  r[  r\  INVALID_INPUTr   r  )r   r   r   rT  r  r^  r_  r   serialized_in_specserialized_out_specflat_args_with_pathreceived_specr[  r\  r   flat_example_inputss                   r   _aoti_flatten_inputsrh    s5    0r"" 	" GH"((##UXX^^%B%BCC((##!HHNN224
&&2))11G''3**33H 2z""kkG2{##||H;B;N..w7TV+3+?h'R  *0)F)F	|*& 
M9L
MMM>''8
 	
 CVBUQ
1Q4..!D8BU   }7Mi <o
 	
 ? 0B0C	



/A0C
  ''1s   &3I)r   r  r  z.Callable[[Callable[_P, _T]], Callable[_P, _T]])r   r  r   r  r  rs  )r  r   )r  zlist[dict[str, Any]])r   r  r  	list[int])r   rM   r  rs  )r  zCallable[..., None]r  )r&  rM   r   rM   r  rs  )r&  rM   r   rM   rC  r   r  rM   )F)r   rM   rX  r   r  zGenerator[str, None, None])r   rM   rg  r  r  rM   )r   rM   rX  r   r  rM   )r   rM   r{  r   r  rs  )TNN)
r   rM   r  r   r  zOptional[list[str]]r  z)Optional[Callable[[torch.fx.Node], bool]]r  z"tuple[GraphModule, dict[str, int]])r   rM   r  r   )rg  r  r  "AbstractContextManager[None, None])r  r   r  r   r  rj  )r   rM   rg  r  r  r   r  z torch._subclasses.FakeTensorModer   )r  z$Optional[Union[str, dict[str, Any]]]r  zdict[str, Any])r  zGenerator[None, None, None]r  )r   rM   rg  r  rd  r  r  r>   )
r   rM   rg  r  rf  r  rd  r  r  r>   )r<  r  r  r  r  r  )r   )rG  Callable[..., Any]r  r  rB  r  rC  zlist[Optional[str]]r  r   r{  r   r   ztuple[torch.Tensor, ...]r>  zSequence[PlaceholderInfo]r?  ztuple[int, ...]r  rk  )r   torch.Tensorr  rl  )rU  rl  rV  rl  rW  ri  r  rs  )rG  rk  r<  zlist[torch.Tensor]r  r  r  rh  )
r}  rM   r~  rg  rv  r  r  Optional[dict[str, Any]]r  z2Union[list[Union[str, Weights]], str, GraphModule])r  rM   r  r  r  rM   r  r  rv  rk  r  r@   r  r  r  r2   r  z0Callable[[list[object]], Sequence[torch.Tensor]])r  zdict[str, object])r   torch.fx.GraphModuler  zAbstractContextManager[None])r   rM   r  zSequence[object]r   r  r  ztuple[GraphModule, GraphModule])rG  rM   r  r  )r   ztypes.ModuleTyper  r  )r   rM   rg  r  r  r  r  r  r  r  rv  Callable[..., OutputCode]r{  r   r  r>   )
r   rM   rg  r  r  r  rv  ro  r  r>   )r}  rM   r~  r  r  rM   )r}  rM   r~  r  rv  ro  r  rm  r  .Optional[dict[OpOverload, Callable[..., Any]]]r  r   r  CompileFxOutput)r   rM   r~  r  r  r  )r}  rM   r~  r  rv  ro  r  rp  r  r   r  rq  )r   rM   r<  r  r  rk  r  rk  )r   rf   r  rs  )
r   rn  r   z!Union[list[Any], tuple[Any, ...]]r   rm  rT  rm  r  z tuple[list[Any], dict[str, Any]](   
__future__r   r  r  enumrK  r  rM  r  r   r   r  rH  r  abcr   r   collectionsr   r   dataclassesr   inspectr	   r
   operatorr   typingr   r   r   r   r   typing_extensionsr   r   r   r   r   r   unittestr   torch._inductor.async_compiler   torch.fxtorch.utils._pytreer   _pytreer@  functorch.compiler   r   torch._dispatch.pythonr   torch._dynamor   r   r  r   r   torch._dynamo.device_interfacer   torch._dynamo.repro.after_aotr    r8  r!   r"   r#   r$   r%   r&   r'   r(   r)   torch._functorchr!  7torch._functorch._aot_autograd.subclass_parametrizationr*   torch._functorch.aot_autogradr+   r,   r-   r.   torch._inductor.codecacher/   r0   r1   r  r2   r3   r4   r5   !torch._inductor.custom_graph_passr6   torch._inductor.debugr7   r8   torch._inductor.output_coder9   r:   r;   r<   r=   r>   'torch._inductor.runtime.cache_dir_utilsr?   torch._inductor.utilsr@   rA   rB   rC   rD   rE   rF   rG   rH   "torch._library.fake_class_registryrI   torch._library.opaque_objectrJ   torch._loggingrK   torch._utils_internalrL   rM   %torch.fx.experimental.symbolic_shapesrN   rO    torch.fx.passes.fake_tensor_proprP   torch.monitorrQ   torch.utils._ordered_setrR   _dynamo.backends.commonrT   _dynamo.excrU   rV   fx._lazy_graph_modulerW   fx.graphrX   utils._tritonrY   rX  r[   r\   codegen.commonr]   r^   rA  r_   decompositionr`   excra   fx_passes.joint_graphrb   fx_passes.post_gradrc   rd   fx_passes.pre_gradre   r   rf   irrg   rh   output_coderi   triton_bundlerrj   rk   rl   rm   rn   ro   rp   rq   rr   rs   virtualizedrt   collections.abcru   rv   rw   rx   
torch._opsry   )torch.export.pt2_archive._package_weightsrz   r{   r|   r}   r  r   r   torch._inductor.fb.utilstypes&torch._functorch._aot_autograd.schemasr   r   r   r   r  r   r  rq  Enumr   r   r   r   _fx_compile_configr   r2  r   r5  r   r7  r   r   r   _logginggetArtifactLoggerr  r  r  r@  r  r   r   r   r   	lru_cacher   cacher  r,  rR  r]  r_  rn  rx  r  r  r  r  r  r  contextmanagerr  r  r  r  r  r  r  r  r  rB  rN  rR  rX  rA  rt  r  r  r  r  r  r  r  r  r  r  r  r|  r  r  r  r  r  r  r  rh  r   r   r   <module>r     s	   "     	    	 
   # # - !     ? ? U U  $  $ $ A  ;  D =
 
 
 8  O N  B  >
 
 
 @ 7 + ?   W ; & / 2 5 : % & 3 3 U  .  5 B /   ' / )
 
 
  ==:%A$ t_T]((**% L  $v,%,,!778S		ODII    CD ./ $))%// +;; !00<Hnn66xARS ~~77BTU NN44'  ~~77BTU 
4A,'=  T/ / 	
 	
H/VM	M%M8FMMb 38!!+/!!0NN'N N. 38##+/##L+" "15FJ	E(E(E( /E( D	E(
 (E(P*('('(.	(	(!%	('	( ).' "& &	@ <@(8((  y  +
+
'+
 '+
 	+
\ 23QQ'Q -Q 	Q 4Qh
B B# #4R) Rj2Y2Y'2Y
 #2Y -2Y 2Yj  $   J (*) +-.0*,))$) 	)
 &) ) ) () ,) () )XU		  
	 (*_M_M_M %_M &	_MJ )9/3	@+@+$@+ &@+ -	@+
 8@+F qc#c+c c 	c
 &c c c %c 6cL&$''"' ' %	'T $% % %: 0@kk'k  k 	k
 /k -k k kd 0@	++'+ /+ -	+
 +\//*=//j 0@/3EI"WW(W -W -	W
 CW W Wt&9>++(+ -+ C	+
 + +\H9H9(H9 -H9 C	H9
 H9 H9V$ # 	4 # 	,)D (,S(
 )-S(S(
+S( %S(
 &S( &S( S(r   