
    ȅiXJ                   n
   % S r SSKJr  SSKrSSKrSSKrSSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKrSSKrSSKrSSKJrJr  SSKJr  SSKJrJrJrJrJrJr  SSKJr  SSKJrJrJrJ r J!r!J"r"  SS	K#J$r$J%r%  SSK&r&SSK'r&SSK(r&SSK)J*s  J+r,  SS
K-J.r.  SSK/J0r0  SSK1J2r2  SSK3J4r4  SSK5J6r6  SSK7J8r8J9r9  SSK:J;r;  SSK<J=r=J>r>  SSK?J@r@  SSKAJBrB  SSKCJDrD  SSKEJFrG  SSKHJIrIJJrJ  SSKKJLrL  SSKMJNrNJOrO  SSKPJQrQ  SSKRJFrF  SSKSJTrT  SSKUJVrV  \	R                  " \X5      rY\&R                  R                  r[\&R                  R                  r\\r]\"\]\&R                  R                  4   r_\R                  R                  SS5      rb " S  S!\ 5      rc " S" S#\ 5      rd " S$ S%\ 5      re\!" S&5      rf\"\&R                  R                  R                  \i4   rj " S' S(5      rk\k" 5       rl S       SS) jjrm " S* S+5      rn " S, S-\o5      rp\"\n\p4   rqSS. jrr " S/ S05      rs " S1 S2\5      rt " S3 S4\t5      ru " S5 S6\t5      rv " S7 S8\t5      rw " S9 S:\t5      rx " S; S<\t5      ry\z\S=4   r{ " S> S?\y5      r| " S@ SA\|5      r} " SB SC\|5      r~ " SD SE\|5      r " SF SG\y5      r " SH SI\5      r " SJ SK\5      r " SL SM\5      r " SN SO\t5      r " SP SQ\t5      r " SR SS\t5      r " ST SU5      r " SV SW\ 5      r\GR                   " SX SY5      5       r\GR                   " SZ S[\5      5       r\GR                   " S\ S]\5      5       r\GR                   " S^ S_\5      5       rSS` jrSSa jr S         SSc jjr\SSdSSb4                     SSe jjr\D" 5       rSf\Sg'               SSh jr\" \5      GR,                  Si-  Sj-  r/ rSk\Sl'   \SSdSb4                     SSm jjr\GGR4                  " SbSn9  S           SSo jj5       r  S           SSp jjr\4SbSq.         SSr jjjr\4SbSq.         SSs jjjrSSt jr\GR@                  " Su5      rSSv jrSSw jrSSx jrSSy jrSSz jrSS{ jrSS| jr " S} S~5      rSS jr    S           SS jjr\&GRX                  " 5       SSS.         SS jj5       r\&GR\                  " 5       SS j5       rSS jrSS jrSS jrSS jrSS jr\D" 5       rSf\S'    S       SS jjrSS jrSS jrg)a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)Callable
Collection	GeneratorIterableMappingSequence)Path)AnyNoReturnOptionalProtocolTypeVarUnion)SelfTypeIs)enable_python_dispatcher)counters)is_integer_dtype)unset_fake_temporarily)make_fx)guard_or_falsestatically_known_true)	_get_attr)immutable_dictimmutable_list)GraphTransformObserver)preserve_node_meta)
OrderedSet   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_type#TORCHINDUCTOR_PATTERN_MATCH_BACKENDinductorc                  *    \ rS rSr% S\S'   SS jrSrg)SearchFn]   str__name__c                    g N selfargskwargss      Y/home/james-whalen/.local/lib/python3.13/site-packages/torch/_inductor/pattern_matcher.py__call__SearchFn.__call__`       #    r8   Nr;   r   r<   r   returnr   )r5   
__module____qualname____firstlineno____annotations__r>   __static_attributes__r8   rA   r=   r2   r2   ]   s
    M=rA   r2   c                      \ rS rSrSS jrSrg)	ReplaceFnc   c                    g r7   r8   r9   s      r=   r>   ReplaceFn.__call__d   r@   rA   r8   NrB   r5   rD   rE   rF   r>   rH   r8   rA   r=   rJ   rJ   c   s    =rA   rJ   c                  .    \ rS rSr        SS jrSrg)TraceFng   c                    g r7   r8   )r:   fnr;   r<   s       r=   r>   TraceFn.__call__h   s    "rA   r8   N)rS   zUnion[SearchFn, ReplaceFn]r;   r   r<   r   rC   torch.fx.GraphModulerN   r8   rA   r=   rP   rP   g   s'    #,#58#DG#	#rA   rP   Tc                      \ rS rSrSS jrSrg)Multiples   c                8    S[        5       ;  d  U [        L d   eg g )NMULTIPLE)globalsr[   r:   s    r=   __init__Multiple.__init__t   s    *dh.>>>.>*rA   r8   NrC   None)r5   rD   rE   rF   r^   rH   r8   rA   r=   rX   rX   s   s    ?rA   rX   c                   SSK JnJn  [        R                  R
                  S:X  aw  U R                  S/ 5      R                  5       nUR                  U" XUR                  5      5        U R                  S UR                  R                  5        5       5        XPS'   O0U R                  S UR                  R                  5        5       5        SUR                  ;   a  UR                  S   U S'   g g )Nr   )
NodeSourceNodeSourceActionr,   	from_nodec              3     #    U  H4  u  pU[         R                  R                  R                  ;   d  M/  X4v   M6     g 7fr7   torchfxproxy_COPY_META_FIELDS.0kvs      r=   	<genexpr>!_transfer_meta.<locals>.<genexpr>   3      
-EHHNN444 QF-   .>
>c              3     #    U  H4  u  pU[         R                  R                  R                  ;   d  M/  X4v   M6     g 7fr7   rg   rl   s      r=   rp   rq      rr   rs   stack_trace)torch.fx.tracebackrc   rd   r%   traceprovenance_tracking_levelgetcopyappendREPLACEupdatemetaitems)new_metaold_node	pass_namerc   rd   new_from_nodes         r=   _transfer_metar   }   s     @
 ||--2 ["5::<Z=M=U=UVW 
 ++-
 	

 !. 
 ++-
 	

 %"*--"> &rA   c                    ^  \ rS rSr% SrS\S'   S\S'   S\S'   S	\S
'   S\S'   S\S'   S\S'     S         SU 4S jjjr\SS j5       rSS jr	S S jr
S!S jrS"S jrS#S jrS$S jr      S%S jr  S&         S'S jjrSrU =r$ )(Match   ao  
Represents a successfully matched pattern.

The `Match` object is returned to represent a successfully matched
pattern. Included in the Match are the pattern that was matched, the graph
nodes matched, and any args that were used during the matching.

The args and kwargs are specific to the type of pattern that was matched and
provide hints about what was matched.
PatternExprpattern	list[Any]r;   dict[str, Any]r<   list[torch.fx.Node]nodesz'dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxzOptional[torch.fx.GraphModule]replacement_graphc                   > [         TU ]  5         X l        [        U=(       d    / 5      U l        U=(       d    0 U l        / U l        0 U l        Xl        S U l	        g r7   )
superr^   r   listr;   r<   r   r   r   r   )r:   r   r   r;   r<   	__class__s        r=   r^   Match.__init__   sK     	$	l
!%rA   c                .    U R                   R                  $ r7   )r   graphr]   s    r=   r   Match.graph   s    xx~~rA   c                B   U R                   (       az  [        U R                   R                  5       5      [        UR                   R                  5       5      -   H/  nU R                   U   UR                   U   :w  d  M%  [        SU5      e   U R                  R                  UR                  5        U R                  R                  UR                  5        U R                   R                  UR                   5        U R                  R                  UR                  5        g )Nzkwarg mismatch: {})	r<   r"   keysFailedMatchr;   extendr   r}   r   )r:   otherkeys      r=   r   Match.extend   s    ;;!$++"2"2"45
5<<CTCTCV8WW;;s#u||C'88%&:C@@ X 			$

%++&5<<(EMM*rA   c                p    U R                   (       a  [        U R                   5      /U l         U $ / U l         U $ r7   )r;   tupler]   s    r=   bundleMatch.bundle   s.    *.))U499%&	 :<	rA   c                <    SU R                    SU R                   S3$ )NzMatch(..., , )r;   r<   r]   s    r=   __repr__Match.__repr__   s    TYYKr$++a88rA   c                    U R                   n[        U R                  5       H:  nUR                  (       a  M  UR                  (       a  M)  UR                  U5        M<     g r7   )r   reversedr   _erasedusers
erase_node)r:   r   ns      r=   erase_nodesMatch.erase_nodes   s=    

$**%A999QWWW  # &rA   c                    U R                   R                   Vs/ s H!  nUb  U R                   R                  U   OS PM#     sn$ s  snf r7   )r   outputspattern_to_node)r:   ps     r=   output_nodesMatch.output_nodes   sK     XX%%
% -.MTXX%%a(tC%
 	
 
s   (Ac                B    [        S U R                  5        5       5      $ )Nc              3  6   #    U  H  o(       d  M  Uv   M     g 7fr7   r8   )rm   r   s     r=   rp   $Match.output_node.<locals>.<genexpr>   s     82!aAA2s   
	)nextr   r]   s    r=   output_nodeMatch.output_node   s    8t002888rA   c                X    [         R                  X R                  R                  X5        g r7   )ReplacementPatternEntryreplace_with_graphr   r   )r:   r   r;   s      r=   r   Match.replace_with_graph   s      	 22((.."3	
rA   c                  ^ SSK JnJn  [        UR                  U5      (       a  UR                  c  UR                  O[
        R                  " 5       nSS jnU   Uc  [        R                  " [        US9nU" U R                  5      (       Ga}  0 mU R                  S   R                  S   u  p0 U
En
[        U R                  5      U R                  pSU4S jjn[        R                   R"                  R%                  XU4X45        [        R&                  R)                  UU4S j5      nU" X5      n[        R&                  R)                  US	 5      nU" X5      n[+        UR,                  R                  5      [+        UR,                  R                  5      :X  d   e[/        UR,                  R                  UR,                  R                  5       H4  u  nnSUR                  ;   d  M  UR                  S   UR                  S'   M6     O)[        R&                  R)                  US
 5      nU" X5      n[+        U R                  5      S:X  a?  UR,                  R                   H%  n[1        UR                  U R                  S   SS9  M'     [2        R5                  U U R6                  R,                  UU5        SSS5        g! , (       d  f       g= f)zReplace with a graph generated by tracing the replacement_fn.

Args:
    run_functional_passes (bool). If we should run passes that
        assume functional IR (like DCE, remove_noop_ops), on the
        replacement graph.

r   )NullHandlerVNc                P   [        U 5      S:w  a  gU S   nSUR                  ;  a  gUR                  [        [        R
                  R                  R                  [        R
                  R                  R                  [        R
                  R                  R                  /5      ;   $ )Nr,   Fr   eager_input_vals)
lenr~   targetr"   rh   opshigher_order triton_kernel_wrapper_functionalauto_functionalizedauto_functionalized_v2)r   nodes     r=   !should_propagate_eager_input_valsCMatch.replace_by_example.<locals>.should_propagate_eager_input_vals  s~    5zQ8D!2;;*II**KKII**>>II**AA#  rA   )run_functional_passesr   c                d   > [        U [        R                  R                  5      (       a  UTU '   g g r7   )
isinstancerh   ri   Node)r   valnode_to_vals     r=   record(Match.replace_by_example.<locals>.record&  s'    !$66,/D) 7rA   c                   > TU    $ r7   r8   )argr   s    r=   <lambda>*Match.replace_by_example.<locals>.<lambda>.  s
    +cBRrA   c                     U R                   S   $ Nr   r~   r   s    r=   r   r   4  s    #((5/rA   c                ^    SU R                   ;   a  U R                   S   $ U R                   S   $ )Nr   example_valuer   r   s    r=   r   r   J  s1    ( !$ !3/2!3rA   r,   replace_by_exampler   r   r   )r   r   rC   bool)r   torch.fx.Noder   r   rC   ra   )torch._inductor.virtualizedr   r   r   	fake_mode
contextlibnullcontext	functoolspartialfwd_onlyr   r~   r   r;   r<   rh   utils_pytreetree_mapri   map_argr   r   zipr   r   r   r   )r:   replacement_fnr;   trace_fnr   r   r   contextr   	fake_argsfake_kwargs
match_argsmatch_kwargsr   example_valsgraph_with_eager_valsreplacementr   new_noder   r   s                       @r=   r   Match.replace_by_example   s`    	? q{{K88Q[[=P KK'') 		 $,,4I 1<< !)-A););<N)O&	-o+0+;T[[L0 ##,,68P  %xx//6RS )1(N%  %xx//6QR&'<K 066<<=%%++B    +.)//55{7H7H7N7N+&Hh *X]]:<DMM.=&89	+  %xx//3  '~D4::!#$**00A"!"!%A"6 1 $66	K WWs   FK+CK
K)r;   r   r<   r   r   r   r   NN)
r   r   r   r   r;   zOptional[Sequence[Any]]r<   zOptional[dict[str, Any]]rC   ra   )rC   torch.fx.Graph)r   r   rC   ra   )rC   r   rC   r4   r`   )rC   zlist[Optional[torch.fx.Node]])rC   r   )r   r  r;   Sequence[Any]rC   ra   NT)
r   rJ   r;   r  r   zOptional[TraceFn]r   r   rC   ra   )r5   rD   rE   rF   __doc__rG   r^   propertyr   r   r   r   r   r   r   r   r   rH   __classcell__r   s   @r=   r   r      s   	 
O44	55 )-+/&& & &	&
 )& 
& &&  +
9$
9
!/
7D
	
 '+&*p!p p $	p
  $p 
p prA   r   c                  B    \ rS rSr% SrS\S'   S
S jrSS jrSS jrSr	g	)r   i_  zs
Represents a unsuccessful match.

The `FailedMatch` object is returned to represent a failure to match a
pattern.
r4   format_stringc                b    Xl         [        U5      S:  a  [        SU 35      eX l        X0l        g )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r
  r   RuntimeErrorr;   r<   )r:   r
  r;   r<   s       r=   r^   FailedMatch.__init__i  s<    * }#hivhwx  	rA   c                b    U R                   R                  " U R                  0 U R                  D6$ r7   )r
  formatr;   r<   r]   s    r=   __str__FailedMatch.__str__t  s&    !!(($))Ct{{CCrA   c                    gNFr8   r]   s    r=   __bool__FailedMatch.__bool__w      rA   )r;   r
  r<   N)r
  r4   r;   r   r<   r   rC   ra   r  rC   r   )
r5   rD   rE   rF   r  rG   r^   r  r  rH   r8   rA   r=   r   r   _  s     	DrA   r   c                    [        U 5      $ )zp
TypeIs cannot act on `self`. Thus this function exists to let mypy
recognize FailedMatch.__bool__ as a TypeIs.
)r   )ms    r=   is_matchr  ~  s    
 7NrA   c                  t    \ rS rSr% SrS\S'   S\S'   S\S'   S	\S
'    S       SS jjrSS jrSS jrSr	g)r   i  z;
Internal state needed while running PatternExpr._match().
list[Optional[PatternExpr]]r   z*dict[PatternExpr, Optional[torch.fx.Node]]r   r  r   zlist[NodeOrConstant]exclusive_node_setNc               T    Xl         Uc  0 O
[        U5      U l        X0l        / U l        g r7   )r   dictr   r   r  )r:   r   r   r   s       r=   r^   MatchContext.__init__  s*     %4%<r$BW
"$rA   c                    XR                   ;   a)  U R                   U   U:X  a  [        X5      $ [        S5      $ UR                  X 5      nXR                   ;  d   eU(       a  UOSU R                   U'   U$ )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   r   r   _match)r:   r   r   r  s       r=   matchMatchContext.match  sq    ***##G,4T++"#=>>NN4&2222201tW%rA   c                    U R                   R                  5        VVs0 s H#  u  pUR                  5       (       d  M  Uc  M!  X_M%     snn$ s  snnf r7   )r   r   has_multiple_users)r:   r   r   s      r=   filter_multi_user_patterns'MatchContext.filter_multi_user_patterns  sR     "&!5!5!;!;!=
!=))+ 04 GM!=
 	
 
s   AAA)r  r   r   r   r7   )r   r  r   z*Optional[dict[PatternExpr, torch.fx.Node]]r   r  rC   ra   )r   r   r   NodeOrConstantrC   MatchResult)rC   z dict[PatternExpr, torch.fx.Node])
r5   rD   rE   rF   r  rG   r^   r$  r(  rH   r8   rA   r=   r   r     s^     )(??,,
 GK
%,
% D
%
 
% 

%

rA   r   c                  j    \ rS rSrSr\SS j5       rSS jrSS jrSS jr	      SS jr
SS jrS	rg
)r   i  z#
Base class for types of patterns.
c                    g r7   r8   r:   r   r   s      r=   r#  PatternExpr._match  s    MPrA   c                ~     [        U /UR                  S9R                  X5      $ ! [         a  nUs S nA$ S nAff = fNr   )r   r   r$  r   r:   r   es      r=   r$  PatternExpr.match  s:    	djj9??KK 	H	s   #& 
<7<<c                    gr  r8   r]   s    r=   r'  PatternExpr.has_multiple_users  r  rA   c                4    U R                   R                  S-   $ )Nz())r   r5   r]   s    r=   r   PatternExpr.__repr__  s    ~~&&--rA   c              #  N   #    XR                   ;   a  UR                   U    v   g g 7fr7   )r   r:   r   searcheds      r=   find_anchor_nodesPatternExpr.find_anchor_nodes  s)      &&&%%d++ 's   #%c                ,    [        XR                  5      $ )z
Compare two `PatternExpr`s and return true if they are the
same. Note this is NOT matching a pattern - it is comparing the pattern
structures (for debugging).
)r   r   )r:   r   s     r=   
pattern_eqPatternExpr.pattern_eq  s     %00rA   r8   Nr   r   r   r   rC   r+  r   r   rC   r+  r  r  r   r   r<  OrderedSet[torch.fx.Node]rC   z.Generator[Optional[torch.fx.Node], None, None]r   r   rC   r   )r5   rD   rE   rF   r  r   r#  r$  r'  r   r=  r@  rH   r8   rA   r=   r   r     sH     P P.,,+D,	7,1rA   r   c                  "    \ rS rSrSrSS jrSrg)Argi  zb
Capture an arg which will become an input to the handler.  Args are
passed in depth first order.
c                    [        X U/S9$ )N)r;   r   r.  s      r=   r#  
Arg._match  s    SdV,,rA   r8   Nr   r*  r   r   rC   r+  r5   rD   rE   rF   r  r#  rH   r8   rA   r=   rH  rH    s    
-rA   rH  c                  6    \ rS rSrSrSS jrS	S jrS
S jrSrg)Ignoredi  z,
Match an arg, but don't pass it to handler
c                    [        X 5      $ r7   rJ  r.  s      r=   r#  Ignored._match  s    SrA   c                    g)N*r8   r]   s    r=   r   Ignored.__repr__  s    rA   c                    g)Nz	Ignored()r8   )r:   pps     r=   pretty_printIgnored.pretty_print  s    rA   r8   NrL  r  rV  PatternPrettyPrinterrC   r4   )	r5   rD   rE   rF   r  r#  r   rW  rH   r8   rA   r=   rO  rO    s     rA   rO  c                  T   ^  \ rS rSrSrSU 4S jjrS	S jrS
S jrSU 4S jjrSr	U =r
$ )
KeywordArgi  <
Capture a kwarg which will become an input to the handler.
c                .   > [         TU ]  5         Xl        g r7   r   r^   namer:   r`  r   s     r=   r^   KeywordArg.__init__      	rA   c                $    SU R                   < S3$ )NzKeywordArg(r   r`  r]   s    r=   r   KeywordArg.__repr__  s    TYYM++rA   c                .    [        X U R                  U0S9$ )Nr<   )r   r`  r.  s      r=   r#  KeywordArg._match  s    S		4'899rA   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       a    U R
                  UR
                  :H  $ r7   typingcastr   r   r@  r`  r:   r   r   s     r=   r@  KeywordArg.pattern_eq  5    D%(w!%(DTYY%**-DDrA   re  r`  r4   rC   ra   r  rL  rF  r5   rD   rE   rF   r  r^   r   r#  r@  rH   r  r  s   @r=   r\  r\    s#    ,:E ErA   r\  c                  `   ^  \ rS rSr% SrS\S'   S
U 4S jjrSS jrSS jrSU 4S jjr	S	r
U =r$ )ExclusiveKeywordArgi   r]  r4   r`  c                .   > [         TU ]  5         Xl        g r7   r_  ra  s     r=   r^   ExclusiveKeywordArg.__init__  rc  rA   c                $    SU R                   < S3$ )NzExclusiveKeywordArg(r   re  r]   s    r=   r   ExclusiveKeywordArg.__repr__  s    %dii]!44rA   c                    XR                   ;   a  [        S5      $ UR                   R                  U5        [        X U R                  U0S9$ )Nzexclusive arg appears twicerh  )r  r   r{   r   r`  r.  s      r=   r#  ExclusiveKeywordArg._match  sD    )))<==%%d+S		4'899rA   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       a    U R
                  UR
                  :H  $ r7   rk  rn  s     r=   r@  ExclusiveKeywordArg.pattern_eq  rp  rA   re  rq  r  rL  rF  )r5   rD   rE   rF   r  rG   r^   r   r#  r@  rH   r  r  s   @r=   rt  rt     s*     I5:E ErA   rt  c                     ^  \ rS rSr% SrS\S'   S\S'    S     SU 4S jjjr\\SS j5       5       r	SS	 jr
SS
 jrSS jr      SS jrSS jrSS jrSU 4S jjrSrU =r$ )_TargetExpri  z/
Base class for filtering match by node.target
zlist[FnsType]fnszOrderedSet[FnsType]fns_setc                |  >^ [         TU ]  5         [        U5      (       d  [        U[        5      (       a  U/O
[        U5      nU HW  m[        T[        R                  R                  5      (       d  M.  UR                  U4S jTR                  5        5       5        MY     Xl        [        U5      U l        X l        g )Nc              3  <   >#    U  H  n[        TU5      v   M     g 7fr7   )getattr)rm   overloadrS   s     r=   rp   '_TargetExpr.__init__.<locals>.<genexpr>)  s     PX72x00s   )r   r^   callabler   r4   r   rh   _opsOpOverloadPacketr   	overloadsr  r"   r  r   )r:   r  r   rS   r   s      @r=   r^   _TargetExpr.__init__"  s     	}}
3(<(<se$s)B"ejj99::

PPP  !#
rA   c                    g r7   r8   r]   s    r=   op_TargetExpr.op/  s    rA   c                   U R                   S   n[        U[        5      (       d  UR                  n[	        U R                   5      S:  a  SU S3$ U R                   S   [        [        US 5      L a  SU 3$ U R                   S   [        [        US 5      L a  SU 3$ [        U R                   S   [        R                  R                  5      (       a  [        U R                   S   5      $ U$ )Nr   r,   [z, ...]torch.z	operator.)
r  r   r4   r5   r   r  rh   operatorr  
OpOverload)r:   
first_reprs     r=   fns_repr_TargetExpr.fns_repr3  s    XXa[
*c**#,,Jtxx=1zl&))XXa[GE:t<<J<((XXa[GHj$??zl++UZZ%:%:;;txx{##rA   c                    U R                   [        L a  SnO#U R                   S:w  a  SU R                    S3nOSnU R                  R                   SU R	                  5        U S3$ )Nz
, MULTIPLEr,   r   r    ()r   r[   r   r5   r  )r:   comma_userss     r=   r   _TargetExpr.__repr__C  s^    ::!&KZZ1_tzzl!,KK..))*!DMMO+<[MKKrA   c                b    [        U R                  [        5      =(       d    U R                  S:  $ )Nr,   )r   r   rX   r]   s    r=   r'  _TargetExpr.has_multiple_usersL  s     $**h/A4::>ArA   c                    [         er7   NotImplementedErrorr;  s      r=   r=  _TargetExpr.find_anchor_nodesO  s
     "!rA   c                    [        U[        R                  R                  5      =(       a8    UR                  U R                  :H  =(       a    [        U5      U R                  ;   $ r7   )r   rh   ri   r   r  extract_targetr  )r:   r   s     r=   
_match_fns_TargetExpr._match_fnsT  sD    tUXX]]+ 5477"5t$4	
rA   c                    XR                   ;   =(       d;    U R                  [        L =(       d"    [        UR                  5      U R                  :H  $ r7   )r   r   r[   r   r.  s      r=   _match_users_TargetExpr._match_users[  s;    KK -zzX%-4::$**,	
rA   c                  > [         R                  " [        U5      n[        TU ]  U5      =(       aY    U R
                  UR
                  :H  =(       a9    U R                  UR                  :H  =(       a    U R                  UR                  :H  $ r7   )rl  rm  r   r   r@  r  r  r   rn  s     r=   r@  _TargetExpr.pattern_eqb  sf    D%(Gu% *588#*EII%* 

ekk)		
rA   )r  r  r   )r,   )r  z!Union[FnsType, Sequence[FnsType]]r   zUnion[Multiple, int]rC   ra   r  r  rD  r   r   rC   r   )r   r   r   r   rC   r   rF  )r5   rD   rE   rF   r  rG   r^   r  r   r  r  r   r'  r=  r  r  r@  rH   r  r  s   @r=   r~  r~    s     
   UV4=Q	     LB""+D"	7"



 
rA   r~  .c                     ^  \ rS rSrSrSS.         SU 4S jjjr\      SS j5       r\      SS j5       rSS jr	SS	 jr
SS
 jr      SS jrSU 4S jjrSrU =r$ )_TargetArgsExprio  z=
Base class for filtering match by node.{target,args,kwargs}
r,   )_usersc               r  > [         TU ]  X5        [        U5      U l        [	        U5      U l        [        S [        R                  " X4R                  5       5       5       5      (       a  U R                  U l        OU R                  U l        U R                  U R                  U R
                  5      U l        g )Nc              3  X   #    U  H   n[        U[        [        [        45      v   M"     g 7fr7   )r   r   r   r   rm   xs     r=   rp   +_TargetArgsExpr.__init__.<locals>.<genexpr>~  s(      
; q4u-..;s   (*)r   r^   r   r;   r   r<   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)r:   r  r  r;   r<   r   s        r=   r^   _TargetArgsExpr.__init__t  s     	%$K	6l 
__T==?;
 
 
  ..DL..DL $TYY DrA   c                l    / U QUR                  5       Q7n[        U 5      /UR                  5       Q7nX#4$ r7   )r  r   r   )r;   r<   r  specs       r=   r  _TargetArgsExpr.simple_flatten  s8     +4*&--/*D	*FKKM*|rA   c                   ^^ [         [        [        [        [        [        0mSUU4S jjm[
        R                  " TX4U4S jS9n[
        R                  " U5      u  p4X44$ )Nc                   > [        U 5      nTR                  U5      nUb   [        R                  " TU" U 5      U4S jS9$ U $ )Nc                    > [        U 5      T;   $ r7   typer  type_mappings    r=   r   F_TargetArgsExpr.pytree_flatten.<locals>.convert_type.<locals>.<lambda>  s    d1g&=rA   is_leaf)r  ry   pytreer   )r  cls
convert_fnconvert_typer  s      r=   r  4_TargetArgsExpr.pytree_flatten.<locals>.convert_type  sG    q'C%))#.J% qM= 
 HrA   c                    > [        U 5      T;   $ r7   r  r  s    r=   r   0_TargetArgsExpr.pytree_flatten.<locals>.<lambda>  s    d1g5rA   r  )r  r   rC   r   )r   r   r   r   r   r  r   tree_flatten)r;   r<   normalized_args_treeflatr  r  r  s        @@r=   r  _TargetArgsExpr.pytree_flatten  s^    
 E%D*
		 		  &N5 

 (()=>
zrA   c                   U R                  5       /[        [        U R                  5      QU R                  R                  5        VVs/ s H  u  pU SU 3PM     snnQnU R                  [        L a  UR                  S5        O.U R                  S:w  a  UR                  SU R                   35        U R                  R                   SSR                  U5       S3$ s  snnf )N=_users=MULTIPLEr,   _users=r  r   r   )r  mapreprr;   r<   r   r   r[   r{   r   r5   join)r:   rn   ro   r;   s       r=   r   _TargetArgsExpr.__repr__  s    MMO
tyy!
 &*[[%6%6%89%8TQ1QCj%89

 ::!KK)*ZZ1_KK'$**./..))*!DIIdO+<A>> :s   Cc           
       ^ U R                  5       /U4S jU R                   5       QU R                  R                  5        VVs/ s H  u  p#U STR	                  U5       3PM     snnQnU R
                  [        L a  UR                  S5        O.U R
                  S:w  a  UR                  SU R
                   35        SnU R                  R                   SUR                  U5       S3$ s  snnf )	Nc              3  F   >#    U  H  nTR                  U5      v   M     g 7fr7   )rW  )rm   r  rV  s     r=   rp   /_TargetArgsExpr.pretty_print.<locals>.<genexpr>  s     4)Qbooa  )s   !r  r  r,   r  r   r  r   )r  r;   r<   r   rW  r   r[   r{   r   r5   r  )r:   rV  rn   ro   r;   
joiner_strs    `    r=   rW  _TargetArgsExpr.pretty_print  s    MMO
4$))4
 7;kk6G6G6IJ6Ida1R__Q'()6IJ

 ::!KK)*ZZ1_KK'$**./
..))*!JOOD,A+B!DD Ks   "C,c                   U R                  U5      (       a,  [        UR                  5      [        U R                  5      :w  a  [        SX5      $ U R	                  X5      (       d  [        SU 5      $ UR                  nUR
                  n[        U5      [        U R
                  5      :  a  SSKJn  [        UR                  5      (       d   eU" UR                  UR                  UR
                  5      nUc  [        SX5      $ Uu  p4[        U5      [        U R                  5      :X  aI  [        U5      [        U R
                  5      :  a'  U Vs0 s H  owU R
                  ;   d  M  XtU   _M     nnO2[        SX5      $ U Vs0 s H  owU R
                  ;   d  M  XtU   _M     nnU R                  X45      u  pU R                  u  pX:w  a  [        SX5      $ [        U5      [        U
5      :X  d   e[        X 5      n[        X5       H  u  p[        U[        5      (       a8  UR!                  X5      n[#        U5      (       d  Us  $ UR%                  U5        MR  [        U[&        R(                  R*                  5      (       d  X:w  d  M  [        SUUUS9s  $    UR,                  R/                  U5        UR                  UR0                  U '   U$ s  snf s  snf )Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionzargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})r   )r  r   r;   r   r  r<   torch.fx.operator_schemasr  r  r   r  r  r   r   r   r   r$  r  r   rh   ri   r   r   r{   r   )r:   r   r   _args_kwargsr  normalized_args_and_kwargsi
node_items	node_spec
self_items	self_specr  r   
child_nodechild_matchs                   r=   r#  _TargetArgsExpr._match  s[   t$$DII#dii.(HGTT  ++2D99		++w<#dkk**DDKK(((();TYY*& *1"#KTXX!;u:TYY/CLCDT4T6=RgdkkAQ}q!*}gGRG&@$  /6Jgdkk9I}q!*}gGJ $U <
 $ 5 5
!5yLL:#j/111##&z#>G';//!ii<,,&&%J66*:O"9#	  $? 	
t++		$= S Ks   K'	KK	Kc              #    #    XR                   ;   a  UR                   U    v   gU R                  S    H  n[        U[        5      (       d  M  UR	                  X5       Hu  n[        U[
        R                  R                  5      (       d  M.  UR                   H7  nXR;  d  M
  U R                  U5      (       d  M"  Uv   UR                  U5        M9     Mw     M     g7f)z
This is used when we are matching a pattern with multiple outputs.
There is a partial match (stored in ctx) and we want to walk
this pattern to find a connection to an already-matched node.

Yields candidate nodes that `self._match` might like.
Nr   )r   r  r   r   r=  rh   ri   r   r   r  add)r:   r   r<  r   
other_noder   s         r=   r=  !_TargetArgsExpr.find_anchor_nodes  s      &&&%%d++,,Q/G';//")";";C"JJ%j%((--@@  * 0 0/#t44&*
 (T 2	 !1 #K 0s   ACAC"C:"Cc                  > [         R                  " [        U5      n[        TU ]  U5      =(       a\    U R
                  S   UR
                  S   :H  =(       a6    [        S [        U R
                  S   UR
                  S   5       5       5      $ )Nr,   c              3  z   #    U  H1  u  p[        U[        5      (       a  UR                  U5      OX:H  v   M3     g 7fr7   r   r   r@  rm   abs      r=   rp   -_TargetArgsExpr.pattern_eq.<locals>.<genexpr>  s4      TDA $.a#=#=Q16IT   9;r   )rl  rm  r   r   r@  r  allr   rn  s     r=   r@  _TargetArgsExpr.pattern_eq  s    D%(Gu% %%a(E,B,B1,EE  5 5a 8%:P:PQR:ST 	
rA   )r;   r  r  r<   )
r  z/Union[torch.fx.node.Target, str, Sequence[Any]]r;   r   r  zUnion[int, Multiple]r<   r   rC   ra   )r;   r  r<   zMapping[Any, Any]rC   z9tuple[Sequence[Any], Union[_SimpleSpec, pytree.TreeSpec]]r  rY  rB  rD  rF  )r5   rD   rE   rF   r  r^   staticmethodr  r  r   rW  r#  r=  r@  rH   r  r  s   @r=   r  r  o  s     ()	E<E E %	E
 E 
E E& %6	B  %6	B 8
?E4l33+D3	732	
 	
rA   r  c                      \ rS rSrSrSrSrg)CallFunctioni!  zJ
Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
call_functionr8   Nr5   rD   rE   rF   r  r  rH   r8   rA   r=   r  r  !  s     
BrA   r  c                      \ rS rSrSrSrSrg)
CallMethodi)  zO
Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
call_methodr8   Nr  r8   rA   r=   r  r  )       
BrA   r  c                      \ rS rSrSrSrSrg)
CallModulei1  zH
Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
call_moduler8   Nr  r8   rA   r=   r  r  1  r  rA   r  c                  "    \ rS rSrSrSS jrSrg)_TargetExprVarArgsi9  zS
Matches a call_function node with any arguments which are passed into the pattern
c                   U R                  U5      (       d  [        S5      $ U R                  X5      (       d  [        S5      $ [        X 5      nUR                  R                  U5        UR                  UR                  U '   UR                  R                  UR                  5        UR                  R                  UR                  5        U$ )Nfunction_mismatchmultiple_users)r  r   r  r   r   r{   r   r   r;   r   r<   r}   )r:   r   r   r  s       r=   r#  _TargetExprVarArgs._match>  s    t$$233  ++/00#	t++		$	dii 	$rA   r8   NrB  rM  r8   rA   r=   r
  r
  9  s    rA   r
  c                      \ rS rSrSrSrg)CallFunctionVarArgsiM  r   r8   Nr5   rD   rE   rF   r  rH   r8   rA   r=   r  r  M  s    	BrA   r  c                      \ rS rSrSrSrg)CallMethodVarArgsiQ  r  r8   Nr  r8   rA   r=   r  r  Q      	BrA   r  c                      \ rS rSrSrSrg)CallModuleVarArgsiU  r  r8   Nr  r8   rA   r=   r  r  U  r  rA   r  c                  X   ^  \ rS rSrSrSS	U 4S jjjrS
S jrSS jrSU 4S jjrSr	U =r
$ )ListOfiY  z
Matches a repeated pattern
c                h   > [         TU ]  5         [        U[        5      (       d   eXl        X l        g r7   )r   r^   r   r   r   r   )r:   r   r   r   s      r=   r^   ListOf.__init__^  s+    ';////rA   c                N    U R                   R                   SU R                   S3$ Nr  r   )r   r5   r   r]   s    r=   r   ListOf.__repr__d  $    ..))*!DLL>;;rA   c                J   [        U[        [        45      (       a  [        U5      S:X  a  [	        S5      $ [        X 5      nUR                  5       nSn[        U5       H  u  pg[        UR                  XGR                  S9nUR                  U R                  U5      n	UR                  5       n[        U	5      (       d!  U R                  (       d  [	        SXi5      s  $ M  SnUR                  U	R!                  5       5        M     U(       d  [	        S5      $ UR!                  5       $ )Nr   non_listFr2  zlist[{}]: {}Tzlist: no_match)r   r   r   r   r   r   r(  	enumerater   r   r   r$  r   r  r   r   r   )
r:   r   r   r  r   matchedr  r  	child_ctxr  s
             r=   r#  ListOf._matchg  s    $u..#d)q.z**# 88:&t_MA$_4D4DI $//$,,
CK'BBDOK((||&~qFFGHH['')* - /00xxzrA   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       aE    U R
                  R	                  UR
                  5      =(       a    U R                  UR                  :H  $ r7   )rl  rm  r   r   r@  r   r   rn  s     r=   r@  ListOf.pattern_eq  sU    D%(Gu% .''6.-	
rA   )r   r   F)r   r   r   r   rC   ra   r  )r   r   r   r   rC   r+  rF  rr  r  s   @r=   r  r  Y  s&     <0
 
rA   r  c                     ^  \ rS rSr% S\S'   SU 4S jjr\SS j5       rSS jrSS jr	SS jr
      SS	 jrSS
 jrSU 4S jjrSrU =r$ )MultiOutputPatterni  r  r   c                   > [         TU ]  5         [        US   [        5      (       d   e[	        S U 5       5      (       d   U5       e[        U5      U l        US   R                  U l        g )Nr   c              3  V   #    U  H  oS L =(       d    [        U[        5      v   M!     g 7fr7   )r   r   r  s     r=   rp   .MultiOutputPattern.__init__.<locals>.<genexpr>  s!     LGq9:
1k ::Gs   '))r   r^   r   r~  r  r   r   r  )r:   r   r   s     r=   r^   MultiOutputPattern.__init__  s[    '!*k2222LGLLLUgULG}!*--rA   c                j    [         R                  " [        U R                  S   5      nUR                  $ Nr   )rl  rm  r~  r   r  )r:   outputs     r=   r  MultiOutputPattern.fns  s&     [$,,q/:zzrA   c                N    U R                   R                   SU R                   S3$ r  )r   r5   r   r]   s    r=   r   MultiOutputPattern.__repr__  r  rA   c                    U R                    Vs/ s H  o!R                  U5      PM     nnSS 3nU R                  R                   SUR	                  U5       3nU S3nU$ s  snf )Nz,
z  z([z
]))r   rW  r   r5   r  )r:   rV  r  r;   r  str_outs         r=   rW  MultiOutputPattern.pretty_print  sg    ,0LL9Lq"L94&\
^^,,-R
0E/FGIT"	 :s   A!c                >   [         R                  " [        U R                  S   5      nUR	                  X15      n[        U5      (       d  U$ U R                  SS   H>  nUc  M  U R                  XR5      n[        U5      (       d  Us  $ UR                  U5        M@     U$ )Nr   r,   )rl  rm  r~  r   r$  r  _match_from_anchorsr   )r:   r   r   r0  r  r   r  s          r=   r#  MultiOutputPattern._match  s    [$,,q/:IIf#{{H||AB'G227@KK((""HH[! ( rA   c                    [        UR                  5      n[        S5      nUR                  U[	        5       5       H8  nUR                  X5      n[        U5      (       a  Us  $ [        U5      Ul        M:     U$ )Nzno anchor found)r   r   r   r=  r"   r$  r  )r:   r   r   priorr  r   s         r=   r8  &MultiOutputPattern._match_from_anchors  sh     S(()$%67--c:<@D		'(A{{"&u+C A rA   c                     [        U R                  UR                  S9R                  X5      $ ! [         a  nUs S nA$ S nAff = fr1  )r   r   r   r$  r   r3  s      r=   r$  MultiOutputPattern.match  s<    	DJJ?EEdQQ 	H	s   ,/ 
AA A Ac                *  > [         R                  " [        U5      n[        TU ]  U5      =(       ab    [        U R                  5      [        UR                  5      :H  =(       a0    [        S [        U R                  UR                  5       5       5      $ )Nc              3  z   #    U  H1  u  p[        U[        5      (       a  UR                  U5      OX:H  v   M3     g 7fr7   r  r  s      r=   rp   0MultiOutputPattern.pattern_eq.<locals>.<genexpr>  s4      <DA $.a#=#=Q16I<r  )	rl  rm  r   r   r@  r   r   r  r   rn  s     r=   r@  MultiOutputPattern.pattern_eq  sm    D%(Gu% DLL!S%77 emm< 	
rA   )r  r   )r   zSequence[Optional[PatternExpr]]rC   ra   )rC   z-Union[Callable[..., Any], str, Sequence[Any]]r  rY  rB  )r   r   r   r   rC   r+  rC  rF  )r5   rD   rE   rF   rG   r^   r  r  r   rW  r#  r8  r$  r@  rH   r  r  s   @r=   r)  r)    s]    ((   
< ")5		
 	
rA   r)  c                  ^   ^  \ rS rSrSrSU 4S jjr\S	S j5       rS
S jrSU 4S jjr	Sr
U =r$ )RepeatedExpri  zh
Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
c                P   > [         TU ]  5         Xl        UR                  U l        g r7   )r   r^   inner_patternr  )r:   rF  r   s     r=   r^   RepeatedExpr.__init__  s!    *""rA   c                .    U R                   R                  $ r7   )rF  r  r]   s    r=   r  RepeatedExpr.fns  s    !!%%%rA   c                   UR                  U R                  U5      n[        U5      (       d  U$ UR                  R	                  U R                  5        U R                  R                  U[        U/5      5       HW  n[        U /UR                  S9R                  U R                  U5      n[        U5      (       d  Us  $ UR                  U5        MY     U$ r1  )
r$  rF  r  r   popr=  r"   r   r   r   )r:   r   r   r  anchor_nodeanchor_ms         r=   r#  RepeatedExpr._match  s    IId(($/{{H	
  --??TF#
K $TF$**=CC""KH H%%HHX
 rA   c                   > [         R                  " [        U5      n[        TU ]  U5      =(       a%    U R
                  R	                  UR
                  5      $ r7   )rl  rm  r   r   r@  rF  rn  s     r=   r@  RepeatedExpr.pattern_eq  sF    D%(w!%( 
T-?-?-J-J.
 	
rA   )rF  r  )rF  r~  rC   ra   )rC   zSequence[FnsType]rB  rF  )r5   rD   rE   rF   r  r^   r  r  r#  r@  rH   r  r  s   @r=   rD  rD    s0    #
 & &&
 
rA   rD  c                  l    \ rS rSrSrS	S jr\\R                  S
SS jj5       5       r	SS jr
SS jrSrg)rZ  i  z
Serializes Patterns to executable python.
XXX: currently only used and tested for fuse attention patterns. May not cover
all patterns.
c                z    [         R                  R                  R                  5       U l        0 U l        0 U l        g r7   )rh   ri   r   
_Namespace	namespacememoized_objs_namesmemoized_objs_ppr]   s    r=   r^   PatternPrettyPrinter.__init__   s*    224;= 8:rA   c                (   [        5       n[        U S5      (       d   eU R                  US9nUR                   Vs/ s H$  nUR                  U    SUR                  U    3PM&     nnUR                  U SU 35        SR                  U5      $ s  snf )zE
Serializes obj to python code with obj written out to `output_name`
rW  )rV  z = 
)rZ  hasattrrW  rU  rV  r{   r  )objoutput_namerV  out_strr   r0  s         r=   runPatternPrettyPrinter.run  s     "#sN++++""b") --
- %%c*+3r/B/B3/G.HI- 	 

 	S	23yy  
s   +Bc                    [        U[        5      (       a5  U R                  R                  U5      =n(       a  U$ U R	                  U5      $ [        US5      (       a  UR                  U 5      $ [        U5      $ )NrW  )r   r  rU  ry   memoizerZ  rW  r  )r:   r[  memoized_names      r=   rW  !PatternPrettyPrinter.pretty_print  sh    c?++ $ 8 8 < <S AA}A$$||C((3''##D))CyrA   c                    UR                  U 5      nUR                  5       nS H  nUR                  US5      nM     U R                  R	                  US 5      nXPR
                  U'   X R                  U'   U$ )N)zaten.r  zprims.r  )rW  r  replacerT  create_namerU  rV  )r:   r[  obj_strobj_nameprefixtmp_names         r=   ra  PatternPrettyPrinter.memoize$  sr    ""4(<<>3F''3H 4 >>--h=(0  %%,c"rA   )rU  rV  rT  Nr`   )r0  )r[  r   r\  r4   rC   r4   )r[  r   rC   r4   )r[  r  rC   r4   )r5   rD   rE   rF   r  r^   r  r   cacher^  rW  ra  rH   r8   rA   r=   rZ  rZ    s6    ;
 __!  !$		rA   rZ  c                  &    \ rS rSr    SS jrSrg)_PassDictsTypei0  c                    g r7   r8   )r:   rn   s     r=   __getitem___PassDictsType.__getitem__1  s     rA   r8   N)rn    tuple[str, torch.fx.node.Target]rC   list[PatternEntry])r5   rD   rE   rF   rp  rH   r8   rA   r=   rn  rn  0  s    !1!	!rA   rn  c                  T    \ rS rSr% S\S'   S\S'   S
S jr  S       SS jjrS	rg)PatternEntryi6  r   r   Callable[[Match], bool]extra_checkc                    [         er7   r  r:   r$  r   r   s       r=   applyPatternEntry.apply;  s    !!rA   Nc                >   UcK  [        U R                  S5      (       d   eU R                  R                   H  nU R                  XUS9  M     g [	        U[
        [        45      (       ay  [        U R                  S5      (       d   eU(       a+  XR                  R                  U4   R                  SU 5        g XR                  R                  U4   R                  U 5        g [        R                  " [        [           U5      nU H  nU R                  XRUS9  M     g )Nr  prependr  r   )rZ  r   r  registerr   r   PatternMatcherPassr  insertr{   rl  rm  r   rn  )r:   
pass_dictsr   r~  rS   r  s         r=   r  PatternEntry.register>  s     >4<<////ll&&jg> '
T+=$>??4<<....LLOOV45<<QELLOOV45<<TBXn%=zJJa9  rA   r8   r$  r   r   r  r   r   rC   ra   r  )r  /Union[_PassDictsType, Sequence[_PassDictsType]]r   z!Union[torch.fx.node.Target, None]r~  r   rC   ra   )r5   rD   rE   rF   rG   rz  r  rH   r8   rA   r=   ru  ru  6  sK    ((" 59	:C: 2: 	:
 
: :rA   ru  c                  *    \ rS rSr% S\S'   SS jrSrg)LoweringPatternEntryiT  Callable[..., Any]handlerc                   [         R                  " U R                  5      " [         R                  " U R                  U5      5      nUR	                  U5         UR                  U[        UR                  5      UR                  5      nUR                  R                  UR                  5        UR                  U5        S S S 5        UR                  S   UL d   eUR                  5         g ! , (       d  f       N3= f)N)r   wrapsr  r   inserting_beforer   r   r;   r<   r~   r}   replace_all_uses_withr   r   )r:   r$  r   r   r  r   s         r=   rz  LoweringPatternEntry.applyX  s    //$,,/	0A0A$,,PU0VW##D)--guUZZ7H%,,WK##DII.&&{3 * {{2$&&& *)s   A'C++
C9r8   Nr  )r5   rD   rE   rF   rG   rz  rH   r8   rA   r=   r  r  T  s    rA   r  c                  .    \ rS rSr% SrS\S'   SS jrSrg)	GraphPatternEntryib  z0
A pattern that runs a function on the FX graph
r  r  c                    UR                  U5         U R                  " U/UR                  Q70 UR                  D6  S S S 5        g ! , (       d  f       g = fr7   )r  r  r;   r<   ry  s       r=   rz  GraphPatternEntry.applyj  s:    ##D)LL<<u||< *))s   +A
Ar8   Nr  )r5   rD   rE   rF   r  rG   rz  rH   r8   rA   r=   r  r  b  s      =rA   r  c                  V    \ rS rSr% SrS\S'   \          S	S j5       rS
S jrSr	g)r   io  z'
The replacement pattern for the graph
zCallable[..., list[Any]]normalize_argsc                0  ^^^^^^ / m " UU4S jS[         R                  R                  5      nU R                  5       n[	        U5      S:X  a  US   nOUS   (       d   e[        US   R                  R                  5      nU Vs/ s HA  n[        U[         R                  R                  5      (       d  M.  UR                  U5      U4PMC     n	n[        U	[        R                  " S5      S9S   n          S
S jmTR                  U5         [        U[         R                  R                  5      (       d   eU" U5      R                   " T6 n
[        U
[         R                  R                  5      (       a  U
/n
SS jm      SUUUUUU4S jjm[	        U5      [	        U
5      :X  a  [#        XZ5       H  u  pT" X5        M     O[	        U5      S:X  d   eT" US   U
5        S	S	S	5        U R%                  5         g	s  snf ! , (       d  f       N$= f)zD
Inserts the replacement graph into the toplevel graph at the match
c                  >   >^  \ rS rSrSrSrSrSU UU4S jjrSrU =r	$ )<ReplacementPatternEntry.replace_with_graph.<locals>.Replaceri  Nc                  > UR                   S;   a  [        TU ]	  U5      $ UR                  nU R	                  U5      u  p4UR                   S:X  Ga  [        U5      (       d   eTR                  X#U5      nTR                  U5        [        UR                  USS9  SUR                  ;   a  UR                  S   UR                  S'   SUR                  ;   a  SUR                  ;  av  UR                  S   UR                  S'   [        UR                  S   [        R                  5      (       a.  SUR                  ;   d   eUR                  S   UR                  S'   U$ UR                   S:X  a  S	S
KJn  [        TU ]=  X#U5      n[        U[        R                   R"                  5      (       d  [%        SU SU S35      eTR&                  c   eS nTR&                  R)                  5        H  u  pXzL d  M  U	n  O   UcG  [        U[*        5      (       d   eU" TR&                  U5      u  pTR&                  R-                  X5        TR                  U5      nTR                  U5        U$ [%        SU 35      e)N)placeholderr0  r   Interpreter_Replacerr   r   r   tensor_metaget_attrr   )unique_graph_name_with_rootzNYI: replacement_graph.z is not a graph module. Got .z
unhandled )r  r   run_noder   fetch_args_kwargs_from_envr  r   r{   r   r~   r   rh   Tensortorch._higher_order_ops.utilsr  r  ri   GraphModuler  owning_modulenamed_modulesr4   register_module)r:   r   r   r;   r<   resultr  sub_gm
graph_namer   mod_getattr_noder   added_replacement_nodesr   s                r=   r  EReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_node  sG   7777 7+D11#>>tD77o-#F++++"00vFF+226:"!'!%"8 *TYY6:>))DV:W$67		)e6;;.F-1YYu-=E*%dii&6EE#0DII#==#=9==9QFKK6!M77j( #W-fFCF%fehh.B.BCC15fX=YZ`Yaabc  !..:::!%J"'"5"5"C"C"E!=)*J! #F "))&#6666(C!//") ++;;JO#(>>*#=L+22<@'')Jtf*=>>rA   r8   r   r   rC   r   )
r5   rD   rE   rF   r  r  r  r  rH   r  )r   r  r   s   @r=   Replacerr    s    KKH9? 9?rA   r  r,   r   )r   c                B   U /n[         [        R                  R                     " 5       nU(       ao  UR	                  5       nXe;  aP  Xc;  aK  [        US5      (       a:  UR                  U5        X&R                  U'   UR                  UR                  5        U(       a  Mn  g g )Nr~   )
r"   rh   ri   r   rK  rZ  r  r~   r   all_input_nodes)r   tag_name	tag_valueinput_stopsqueuevisitedr   s          r=   percolate_tagsBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tags  sx     FE /1Giik&.V,,KK$)2HHX&LL!4!45 %rA   c                    U R                   S:w  a  g U R                  [        R                  :w  a  g [	        U R
                  5      S:X  d   eU R
                  S   $ )Nr   r#   r,   )r  r   r  getitemr   r;   r   s    r=   maybe_getitemAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitem  sJ    77o-;;("2"22499~***yy|#rA   c           	     Z  > SU4S jjnU c  Ub   eg [        U [        R                  R                  5      (       d   eUc;  U R	                  S US9  [        U R                  5      S:X  a  T	R                  U 5        g [        U[        R                  R                  5      (       a  SUR                  ;  a%  UR                  R                  U R                  5        S H4  nX0R                  ;   d  M  T" XU R                  U   [        T5      5        M6     U R	                  XS9  [        U R                  5      S:X  a  T	R                  U 5        g [        U R                  R                  5       5      nU H$  nT
" U5      nUc  [        S5      eT" XQU   5        M&     T	R                  U 5        g )Nc                   > U T;  $ r7   r8   )r   r  s    r=   !filter_nodes_in_newly_added_nodesfReplacementPatternEntry.replace_with_graph.<locals>.replace.<locals>.filter_nodes_in_newly_added_nodes  s      '>>>rA   )delete_user_cbr   r   )	recomputeac_graph_idzPDeleted index from getitem, did you erase the index and not properly replace it?r  )r   rh   ri   r   r  r   r   r   r~   r}   r"   r   r   AssertionError)oldnewr  r  old_usesuseridxr  r;   r   r  r  re  s          r=   re  ;ReplacementPatternEntry.replace_with_graph.<locals>.replace  s{   ?
 ;;&;!#uxx}}5555;--'H .  399~*((-c588==11CHH,1 %A#xx/* #sxx/A:dCS %A -- .  399~*((-:  		 01$D'-C{,n  Dc(+ %   %rA   N)
r   r   r  r4   r  r4   r  rE  rC   ra   r  )r  zUnion[torch.fx.Node, None]r  z3Union[torch.fx.Node, Sequence[torch.fx.Node], None]rC   ra   )rh   ri   Interpreterr   r   r   r   r   r   r   indexminr  
itemgetterr  r  r^  r   r   )r$  r   r   r;   r  r   	last_noder   r   indicesr   r  r  r  r  r  re  s    ` `         @@@@r=   r   *ReplacementPatternEntry.replace_with_graphw  s
    8:>	? >	?uxx++ >	?@ ))+|!$QI?"?a..445E &%Aa/ $Q#%  
 G)<)<Q)?@CI	6	6	6 	6 3		6
 	6( ##I./1E1EFFFF"#45994@K+uxx}}55*m$Q&/Q&HQ& Q& Q&f < C$44 #L >HCC% !? <(A---Q5M /P 	G6 /.s   -H8HCH
Hc           
         UR                   c   eU R                  UUUR                   U R                  " UR                  0 UR                  D65        g r7   )r   r   r  r;   r<   ry  s       r=   rz  ReplacementPatternEntry.applyP  sL    &&222##<u||<		
rA   r8   N)
r$  r   r   r  r   z+Union[torch.fx.Graph, torch.fx.GraphModule]r;   zSequence[torch.fx.Node]rC   ra   r  )
r5   rD   rE   rF   r  rG   r  r   rz  rH   r8   rA   r=   r   r   o  s]     -,VVV GV &	V
 
V Vp
rA   r   c                    gr  r8   )r$  s    r=   _return_truer  Z  s    rA   c                F    [         R                  SU R                  U5        g )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginfor5   )	search_fnr4  s     r=   log_trace_failurer  ^  s    HHJ	rA   Fc                  ^^ [         R                  U 5      mUR                  T5      nU(       d'  UT   R                  U(       a  [	        U5      OS5        gUc#  U(       a  g[
        R                  " SU4S j5        [	        U5      nU H0  mUT:w  a  M  U(       a    g[
        R                  " SUU4S j5        M2     UR                  U5        g)a.  
Check if a pattern is a duplicate. Because we ignore certain types in searching, but not
in matching, use the graph to distinguish equivalent search patterns.

Returns True if a duplicate is found and `skip_duplicates=True` is passed in. Errors if
`skip_duplicates` is False and a duplicate is found.
NFTc                    > ST  S3$ )NDuplicate pattern: z with no graphr8   )pattern_reprs   r=   r   1check_and_add_duplicate_pattern.<locals>.<lambda>  s    ),~FrA   c                    > ST ST  S3$ )Nr  z with duplicated match graph  r8   )	graph_strr  s   r=   r   r    s    ),7TU^T__`arA   )rZ  r^  ry   r{   r4   rh   _check)r   r   seen_patternsskip_duplicatesequiv_pattern_reprsnew_graph_strr  r  s         @@r=   check_and_add_duplicate_patternr  f  s     (++G4L'++L9l#**3u:DI}F	

 JM(	I%a	
 ) }-rA   r8   c
           
     .  ^ ^^^^^^^^ / [         R                  " T 5      R                  R                  5       Qm[         R                  " T 5      (       a  [        T T5      m [         R                  " T5      (       a<  / [         R                  " T5      R                  R                  5       Qn
[        TU
5      mSUUUUUUU UU4	S jjnS	U4S jjnT[        L a  [        R                  " 5       (       a  g[        R                  " SS9   U Vs/ s H0  n[        U[        R                  5      =(       a    UR                  PM2     snmTc  [        T UTTT5      u  pOTnSn[        U[        5      (       a  UOU/ HR  n[        U[         5      (       d  M  [#        UU(       a  UR$                  OSUR&                  U	S9(       d  MJ    SSS5        g   [)        UUUS9nUR+                  U5        UR,                  sSSS5        $ s  snf ! , (       d  f       g= f)
a  
Create a replacement rule based on example functions that get traced
to create patterns.  This supports both training and inference when
run on a joint forward+backward graph.

Args:
    search_fn: traced to give original pattern
    replace_fn: traced to give replacement graph
    example_inputs: example inputs for initial trace
    trace_fn: fwd_only or joint_fwd_bwd
    pass_dict: dict of passes to register to
    extra_check: additional check to run on match(using real shapes)
c           
     	  >	^^ [        T5      nU H-  nX R                  ;  d  M  [        SU SU R                   35      e   [        [        R                  R                  U Vs/ s H  o R                  U   PM     snS 5      5      m/ n[        R                  R                  R                  T5      nUc   eU   [        T5       GH9  u  pV[        TU   [        R                  5      (       d  M*  U(       a'  [        TU   R                  5      (       a
    SSS5        g[        R                  " TU   R                  5       TU   R!                  5       TU   R                  TU   R"                  US9TU'   [$        R&                  " TU   R(                  TU   R!                  5       5       HQ  m[        T[        R*                  5      (       d  M$  [-        U4S jU 5       5      (       d  M@  UR/                  T5        MS     GM<     TnU(       Gd7  U(       Ga  SUU4S jjn T" XT-   5      n	/ n[3        [5        [7        U5      [7        T5      -   5      U	R8                  R:                  5       H  u  p\U[7        U5      :  a  UR/                  UR<                  5        M1  U	R8                  R?                  U5         U	R8                  RA                  X[7        U5      -
     5      nURB                  Ul        URE                  U5        U	R8                  RG                  U5        SSS5        M     X-   nO
 T" TT5      n	[I        U	UTTS	9nU RK                  5       S
   nUc   eURM                  U5      n[N        RP                  RS                  S5      URB                  :X  a#  [T        RW                  SUURX                  UU5        [[        U5      (       a  T" U5      (       az  T" TT5      U l.        [7        U R:                  5      S:X  aI  U R\                  R8                  R:                   H%  n[_        UR`                  U R:                  S
   SS9  M'      SSS5        g SSS5        gs  snf ! [         a  n
[1        TU
5         Sn
A
SSS5        gSn
A
ff = f! , (       d  f       GM4  = f! [         a  n
[1        TU
5         Sn
A
SSS5        gSn
A
ff = f! , (       d  f       g= f)z
Often shapes get burned into the pattern, so our initial match ran with
`ignore_types=(int, ...)`.

Recheck the match with the correct shapes.
z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                     U R                   S   $ r   r   )r   s    r=   r   8register_replacement.<locals>.check_fn.<locals>.<lambda>  s    QVVE]rA   NF)dtypedevicerequires_gradc              3  @   >#    U  H  n[        TU:g  5      v   M     g 7fr7   )r   )rm   r  ro   s     r=   rp   9register_replacement.<locals>.check_fn.<locals>.<genexpr>  s!      ?CKa1!q&998s   c                 >   > T" U [        U 5      [        T5      -
  S  6 $ r7   )r   )args_newr;   r  s    r=   search_fn_new=register_replacement.<locals>.check_fn.<locals>.search_fn_new  s#    ((3x=3t93L3N*OPPrA   )argnamesexclusive_arg_namesscalar_workaroundr   !TORCHINDUCTOR_PATTERN_MATCH_DEBUGz"Specific pattern match: %s%s %s %sr,   r   r   T)r  r   rC   r   )1r   r<   r  rh   ri   r   _dynamor   detect_fake_moder!  r   r  r   r  empty_stridedsizestrider  r  r  shapeSymIntr  r{   r  r   ranger   r   r   r   inserting_afterr  r`  r  r   fx_to_patternr   r$  osenvironry   r  warningr;   r  r   r   r~   )r$  r  r`  sym_argsr   r  gradspecific_patternr  specific_graphr4  sym_arg_namesr  r   r   specific_pattern_matchr   r;   ro   argnames_staticr  rw  
replace_fnr  r  r  search_fn_patternr   s                    @@r=   check_fn&register_replacement.<locals>.check_fn  s?    (D<<'"99A
/RWR^R^Q_a   HH089d#9;R
 (*MM''88>	$$$$]3d1gu||44 0a ? ?$	 Y $11QQ("1gmm#Aw~~&*DG '__T!W]]DGNN<LM%a663 ?CK? < < %OOA.	 N 4*  1#Q Q%)1-D)Q %'M*-c(mc$i78&,,22+ s8},)001C1CD$+11AA+N'5';';'G'G (S]): ;(H /7mmHO'==hG*00;;KH ON+   -7H%)1)T)B
 $1"%(;&7	$  %%'*D###%5%;%;D%A"zz~~ABdiiO8II*$ .//K@V4W4W*2:t*D'u{{#q("44::@@&%&VV%*[[^&3 A G YH I Y :\ ( %))Q7$S YN% ON ( %))Q7$G YB%C Ys   $Q ;2S1%SB,SS+5S!Q%,BS8A'RS1	R#:DSS%
R/R;SRS
R 	S#
S-S9SSS
Sc                   > T Vs/ s H  oR                  U5      PM     nn[        S[        U 5      S-   5       H1  nSU 3U ;  a    O'UR                  U R                  SU 35      5        M3     U (       a   SU < 35       eU$ s  snf )Nr,   	tangents_zleftover kwargs: )rK  r  r   r{   )r<   r`  r;   r  r  s       r=   r  ,register_replacement.<locals>.normalize_args0  s    -<=_T

4 _=q#f+/*A1#f,KK

Yqc?34 + 9.vj99z >s   BFfunctionalize_rng_opsN)r  )r   rw  r  r$  r   rC   r   )r<   r   rC   r   )inspect	signature
parametersr   ismethod_wrap_bound_methodjoint_fwd_bwdrh   is_inference_mode_enabledfunctorch_configpatchr   r  r  gen_pattern_and_search_gmr   r  r  r   r  r   r  r   )r  r  example_inputsr   r  rw  r  r  r  r  replace_argnamesr  r  r  r   gmpattern_matcher_passr  r  s   `` ` ````        @@r=   register_replacementr)    s   2 H)))4??DDFGO	""&y/B	
##MW..z:EEJJLM'
4DE
| || =  **,, 
		e	<ES%
ESJq%,,';AOO;^%
 $3!#KGR (GB %Z::JL ! .0BCC2 "BHH(66$3	  !5 
=	<" M * )

 	$E 
=	<%
 
=	<s1   H7HAH,HH)HH
HzOrderedSet[str]_serialized_patternsc                l   SS jn[         R                  5       (       d  [        S[          35      eUR                  nSSKJn  UR                  " SS9   [        XX45      nS S S 5        [        R                  WU S9n	U[        ;  a  Sn
[        R                  U5        OS	n
U" 5       n[        [         U S
3-  U
5       nU
S:X  a  UR                  U5        OUR                  S5        UR                  U	5        UR                  S5        S S S 5        U$ ! , (       d  f       N= f! , (       d  f       U$ = f)Nc                    [         R                  " S5      n [         R                  " S5      R                  U S9n/ n[        [        R
                  R                  5       Hm  n[        [        R
                  R                  U5      n [        U[        5      (       a0  [        U[        [        45      (       a  UR                  U5        Mk  Mm  Mo     SR                  U5      nSU S3nU U 3$ ! [         a     M  f = f)Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            a               # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor
            import operator

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr  dirrh   	_inductorpattern_matcherr  r   r  
issubclassr   r~  r{   	TypeErrorr  )auto_generated_msgfile_templatepattern_matcher_importsr`  attrformatted_importss         r=   get_file_template-_serialize_pattern.<locals>.get_file_templateo  s    %__
 !
 &'&
( 	  #%778D5??::DADdD))j;4/ / ,2248	/) 9 %MM*ABPQbPccij!2 344  s   AC--
C;:C;z0Could not find serialized patterns directory at r   r$   Fr  )r\  wr  z.pyz

rY  r  )SERIALIZED_PATTERN_PATHis_dirr  r5   torch._functorchr%   r#  gen_patternrZ  r^  r*  r  openwrite)unique_namer  r%  r   r  r:  pattern_namer"  r   serialized_pattern
write_moder6  fs                r=   _serialize_patternrH  h  s   '5R #))++>?V>WX
 	
 %%L;			e	<iU 
= .11'{1S//
  .
%'M	%<.(<<j	IQGGM"GGFO	"#	 
J N) 
=	< 
J	I Ns   D=AD$
D!$
D3	fx_passesserialized_patternszvlist[tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsc
                    [        U5      nS[        R                  ;   a  [        XX4U5      n
O]UR                  n[
        R                  " SU 35      nU(       a  [        X5      (       d  [        R                  SU 5        [        X5      n
[        R                  " U5       H0  n[        U[        5      (       d  M  UR                  c  M)  S Ul        M2     [         R#                  XXGU
45        [%        UUUUUUUUU
U	S9
  g )NPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)r  r  )r   r  r  rH  r5   	importlibimport_modulerZ  r  r	  r  r  	tree_iterr   r)   constantrK  r{   r)  )rC  r  r  r%  r   r  rw  r  r  r  patrD  r  r   s                 r=   gen_register_replacementrS    s     >*N+ N>O
 !))##<\NK
 //KKV a%/c:&&3<<+C
  CL 0  &&	HE 'rA   r  c                r   / [         R                  " U 5      R                  R                  5       QnUc  0 n/ nSnU H5  nX;   a  UR	                  X8   5        M  UR	                  X   5        US-  nM7     U" X5      n	[        U	[        [        [        [        R                  [        R                  4UUUS9U	4$ )Nr   r,   )ignore_typesr  r  r  )r  r  r  r   r{   r  intfloatr   rh   r  r  )
r  r%  r   r  r  r  flat_inputs	input_idxargname	search_gms
             r=   r$  r$     s     A""9-88==?@H KI'09:~89NI  0IudELL%++F/ 3	
 		 	rA   c                "    [        XX#U5      S   $ r/  )r$  )r  r%  r   r  r  s        r=   r@  r@  #  s!     %8@S	 	rA   r}  c               $   ^ ^^^ SUUU U4S jjnU$ )z
Register an aten to inductor IR replacement pattern.  The decorated
function is saved and then called a lowering time allowing direct
pattern to inductor IR conversion.
c                l   > [        U 5      (       d   e[        TTU S9R                  TTS9  SU l        U $ )Nr   rw  r  r}  T)r  r  r  _inductor_lowering_functionr  rw  	pass_dictr   r~  s    r=   	decorator,register_lowering_pattern.<locals>.decorator<  s>        g	

(9g(
..2+rA   r  r  rC   r  r8   r   rw  rb  r~  rc  s   ```` r=   register_lowering_patternrg  /  s      rA   c               $   ^ ^^^ SUUU U4S jjnU$ )z_
Register a pattern that runs a function on the FX graph, allowing
custom transformation code.
c                ^   > [        U 5      (       d   e[        TTU S9R                  TTS9  U $ )Nr_  r}  )r  r  r  ra  s    r=   rc  )register_graph_pattern.<locals>.decoratorS  s6        g	

(9g(
.rA   re  r8   rf  s   ```` r=   register_graph_patternrk  G  s      rA   c                B    U[        [        U R                  5      5      L $ r7   )r   iterr   )r   r   s     r=   is_start_of_fx_graphrn  ]  s    4U[[)***rA   z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)c                    U R                   S:w  a  gU [        R                  R                  R                  R
                  [        R                  R                  R                  R
                  4;   $ )Nr0   F)rT  rh   r   r0   accumulate_grad_defaultresize_storage_bytes_)r  s    r=   "fixme_incorrect_inductor_schema_oprs  g  sW    	||z! 		++33		0088  rA   c                   [        U R                  [        R                  R                  5      (       a:  [        U R                  5      (       d   U R                  R                  R                  $ [        U R                  [        R                  R                  R                  5      (       a  gU R                  S:X  aL  [        U R                  5      (       d   e[        R                  U R                  R                  5      (       a  gOVU R                  S:X  aF  [        U R                  [         5      (       d   e[        R                  U R                  5      (       a  gU R"                  R%                  S5      S L$ )NFr   Tr  out)r   r   rh   r  r  rs  _schema
is_mutable_higher_order_opsauto_functionalizeAutoFunctionalizedr  r  _mutation_op_researchr5   r4   r<   ry   r  s    r=   is_mutation_opr}  s  s   UZZ** 0=={{""---	U,,??RR
 
 ww/!$$$$!!$++"6"677 8	M	!$++s++++!!$++..;;??5!--rA   c                    SU R                   ;   d   eSUR                   ;   d   eU R                   S   UR                   S   :H  $ Nmutation_region_idr   )r  r  s     r=   same_mutation_regionsr    sE    166)))166)))66&'1662F+GGGrA   c                P   UnSUR                   ;  a>  [        X5      (       d.  UR                  nSUR                   ;  a  [        X5      (       d  M.  UR                   R                  SS5      nX!La5  UR                  n[        U5      (       a  US-  nX2R                   S'   X!La  M5  U$ )Nr  r   r,   )r~   rn  prevry   r   r}  )r   r   r   r  s       r=   get_mutation_region_idr    s    A
aff
,5I%5S5SFF aff
,5I%5S5S$8!<
-FF!!#'9#$	 -
 rA   c                X    S[        [        U R                  5      5      R                  ;  $ r  )r   rm  r   r~   r2  s    r=   "should_compute_mutation_region_idsr    s!    tD,='>'C'CCCrA   c                t    SnU R                    H&  n[        U5      (       a  US-  nXR                  S'   M(     g )Nr   r,   r  )r   r}  r~   )r   r  nds      r=   compute_mutation_region_idsr    s9    kk"!#(:$% rA   c                   ^  SU 4S jjnU Vs/ s H2  n[         R                  " U[         R                  R                  5      PM4     nn[         R                  " U5      Ul        U$ s  snf )zI
Wrap a bound method to remove 'self' from its signature for FX tracing.
c                    > T" U 0 UD6$ r7   r8   )r;   r<   rS   s     r=   wrapper#_wrap_bound_method.<locals>.wrapper  s    4"6""rA   rB   )r  	ParameterPOSITIONAL_OR_KEYWORD	Signature__signature__)rS   r  r  r`  paramss   `    r=   r  r    sb    
#
 D 	$ 1 1 G GH   $--f5GNs   9A&c                  \   ^  \ rS rSr  S     SU 4S jjjrS	S jrS
S jrSS jrSrU =r	$ )r  i  c                   > [         TU ]  5         [        [        5      U l        Xl        X l        [        [        5      U l        g r7   )r   r^   r   r   patternsr   	subsystemr  )r:   r   r  r   s      r=   r^   PatternMatcherPass.__init__  s=    
 	  	 #" >I=NrA   c                     U R                   U   $ r7   )r  )r:   items     r=   rp  PatternMatcherPass.__getitem__  s    }}T""rA   c                @   U R                   (       d  g[        U[        R                  R                  5      (       a  UR
                  nOO[        U[        R                  R                  5      (       a  UnUR                  nO[        S[        U5       35      e[        U5      (       a  [        U5        [        R                  " [        U5      nSn/ nSnU R                    H.  u  pxUS:X  a  SnM  UR                  UR!                  XxSS95        M0     U(       a  UR                  UR!                  SSS95        U R"                  b  U R"                  OSn	[        U[        R                  R                  5      (       d   e[%        XU R&                  5         [)        [*        R,                  R/                  U5      SS	9 GH  n
[1        U
5      nU
R2                  S:X  a  U
R2                  U4U R                   ;  a  M=  [5        U
SS
9(       a  MN  U R                   U
R2                  U4    GHU  nU
R6                  (       a    M  UR8                  R;                  U
5      n[=        U5      (       a-  [?        [A        [C        X<RD                  5      5      5      S:w  a  Mp  [F        RH                  RK                  S5      U
RL                  :X  a+  [N        RQ                  SXRR                  XR8                  5        [=        U5      (       d  M  [U        URW                  U5      5      (       d  M  US-  nURY                  XU
5        [Z        [\           S==   S-  ss'   [Z        [\           S==   [?        URD                  5      -  ss'   GMX     GM     S S S 5        U$ ! , (       d  f       U$ = f)Nr   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Fr  T)r  r   sort)r  r  r2  )reverse)allow_cpu_inputsr,   r  z
%s%s %s %spattern_matcher_countpattern_matcher_nodes)/r  r   rh   ri   r  r   Graphr  r  r  r  r  r   r   r  r{   
find_nodesr   r    r  sortedr  r  from_iterabler  r  r.   r   r   r$  r  r   r"   r  r   r  r  ry   r`  r  r	  r;   r   rw  rz  r   backend)r:   r'  r   get_mutation_region_id_partialcountr   has_call_moduler  r   r   r   entryr  s                r=   rz  PatternMatcherPass.apply  s   }}b%((..//HHEEHHNN++E$$B\]abd]e\fg  .e44'.)2):):"E*
& --JB]""&U---OP	 (
 LL))])GH&*nn&@DNNFW	"ehh223333#B4>>By<<UCTR'-77m+(= 
 9PUV!]]DGGV+<=E||++D1A !&s+I77'ST 
 !zz~~&IJdiiWL$		1mmT{{~e6G6G6J'K'K
Ad3 )*ABaGB )*ABc!''lRB) > S CD E CBD s   E'N?N A$N
Nc                8    U R                   R                  5         g r7   )r  clearr]   s    r=   r  PatternMatcherPass.clear  s    rA   )r   r  r  r  r   )r   Optional[str]r  r  rC   ra   )r  rr  rC   rs  )r'  +Union[torch.fx.GraphModule, torch.fx.Graph]rC   rV  r`   )
r5   rD   rE   rF   r^   rp  rz  r  rH   r  r  s   @r=   r  r    sK     $(#'O O !O 
	O O$#?B rA   r  c                     [         er7   r  r   s     r=   _not_implementedr    s    
rA   c                <  ^^^^	^
^ U=(       d    0 nUR                  5        VVs0 s H  u  pVXe_M	     snnm
[        T
5      [        U5      :X  d   e S     SUU
4S jjjm[        R                  " 5       m	 " UU	UUU4S jS[        R
                  R                  5      n[        U [        R
                  R                  5      (       d   eU" U 5      R                  5       n[        U[        5      (       d  [        [        R                  " U5      5      $ U$ s  snnf )z
Convert an FX graph into a PatternExpr.  This is useful for simple
patterns that can only match single functions and fixed-length lists.
c                   > Ub  UOTn[        U [        [        45      (       a  U T;   a  [        TU    5      $ [	        U 5      U;   a
  [        5       $ [        U [        5      (       a(  [        S U  5       5      (       a  U (       a
  [        5       $ U $ )Nc              3  B   #    U  H  n[        U[        5      v   M     g 7fr7   )r   rO  )rm   ys     r=   rp   5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>0  s     &Iq!z!W'='=qs   )r   rW  rV  r\  r  rO  r   r  )r  ignore_types_overridecurrent_ignore_typesrU  inv_scalar_workarounds      r=   process_arg"fx_to_pattern.<locals>.process_arg&  s     &;%F!L 	 a%&&10E+E3A6777**9a3&Iq&I#I#Ia9rA   c                     >^  \ rS rSr\r\r\r        SUUU4S jjr        SUU4S jjr	SU 4S jjr
SrU =r$ )	 fx_to_pattern.<locals>.Converteri6  c                   > [        T5      nU[        T5      :  a  TU   nO<T(       a  UR                  S5      (       d   eUnO[        R                  " SSU5      nUnUT;   a  [        U5      $ [        U5      $ )Ntangentz_\d+$r  )r   r   
startswithresubrt  r\  )	r:   r   r;   r<   r   r`  r  argnumr  s	         r=   r  ,fx_to_pattern.<locals>.Converter.placeholder?  sx     VA3x= {((3333"f5***400!$''rA   c                p  > T	nU[         R                  L a"  [        S T 5       5      4     SU	4S jjjnUn[        R                  " XBU45      u  p#[
        T;   aA  U Vs/ s H
  od" U5      PM     nnUR                  5        VVs0 s H  u  pvXt" U5      _M     nnn[        U/UQ70 UD6$ s  snf s  snnf )Nc              3  :   #    U  H  o[         Ld  M  Uv   M     g 7fr7   )rV  )rm   ts     r=   rp   Afx_to_pattern.<locals>.Converter.call_function.<locals>.<genexpr>`  s      Q#/aC<<s   	c                   > T" X5      $ r7   r8   )r  r  r  s     r=   process_arg_fn_implKfx_to_pattern.<locals>.Converter.call_function.<locals>.process_arg_fn_impl^  s     'q@@rA   r  rV   r  zOptional[Sequence[type[Any]]]rC   zUnion[T, KeywordArg, Ignored])r  r  r   r  r   r   r   r  )
r:   r   r;   r<   process_arg_fnr  r  rn   rU  r  s
           r=   r   .fx_to_pattern.<locals>.Converter.call_functionT  s     )N))) LQ Q#/Q LAA+HA
 3A A "5!??>&>JLD|#3784aq)48;A<<>J>41!^A..>J8888 9Js   !B-B2c                  > [         TU ]  U5      nUR                  S:X  a  [        U[        5      (       ap  UR
                  S   n[        U[        5      (       d   e[        U5      [        U5      :X  d   e[        X#5       H  u  pE[        UR                  5      Ul	        M!     U$ [        UR                  5      Ul	        U$ )Nr0  r   )
r   r  r  r   r   r;   r   r   r   r   )r:   r   rvr;   rr   r   s         r=   r  )fx_to_pattern.<locals>.Converter.run_nodeo  s    !!$BttxJr5$9$9vvay!$
33332w#d)+++!"mFA!#))nAG ,
 I qww<IrA   r8   )r   r4   r;   r  r<   Mapping[str, Any]rC   z&Union[ExclusiveKeywordArg, KeywordArg])r   r4   r;   r  r<   r  rC   r   )r   r   rC   r   )r5   rD   rE   rF   r  r  r  r  r  r   r  rH   r  )r   r  r  r  rU  r  s   @r=   	Converterr  6  s|    &&#	(	(  	( &		(
 4	( 	(*	9	9  	9 &		9
 	9 	96	 	rA   r  r7   r  )r   r   r  r  rh   ri   r  r   r  r^  r   r)  r  tree_leaves)r'  rU  r  r  r  rn   ro   r  r   r  r  r  s    `` `    @@@r=   r  r    s    */R.?.E.E.GH.GdaQT.GH$%->)???? FJ%B	&  __FD DEHH(( DL b%((..////m!Gg{++!&"4"4W"=>>N{ Is   DT)r   get_decomp_fnc               j   [        5          [        5          Ub  U" 5       O	[        5       n[        XSS9" U6 nSSS5        SSS5        SSKJn  U(       a,  U" WR                  5        UR                  R                  5         WR                  5         U$ ! , (       d  f       Na= f! , (       d  f       Nj= f)z>Build a normalized inference graph, for use with fx_to_patternNreal)tracing_moder,   remove_noop_ops)	r   r!   r-   r   fx_passes.post_gradr  r   eliminate_dead_code	recompile)rS   r;   r   r  decompositionsr'  r  s          r=   r   r     s     
"	#%7%9,8MO>Q>S 	 Rf=tD	 &:	# 5! 	$$&LLNI+ &:%9	#	#s!   B$"BB$
B!	B$$
B2c           
       ^ Sm        S
U4S jjn[         R                  R                  S5         [        U S U[	        5       SSS9" U6   SSS5        T(       d   eSSKJn  U" TR                  5        SS	KJ	n  UR                  TR                  5        [         R                  R                  R                  5       TR                  l        TR                  R                  5         TR                  5         T$ ! , (       d  f       N= f)z=Build a normalized training graph, for use with fx_to_patternNc                D   > T(       a   e[        U 5      m[        X40 UD6$ r7   )clone_graphr(   )joint_graphinputsr<   r'  s      r=   record_joint_graph)joint_fwd_bwd.<locals>.record_joint_graph  s&     v% ???rA   c                    [        U 5      $ r7   )r'   )gr  s     r=   r   joint_fwd_bwd.<locals>.<lambda>  s	    +rA   TF)partition_fnr  keep_inference_input_mutations
enable_logr,   r  )early_patterns)r  rU   r  r  r<   r   rC   z1tuple[torch.fx.GraphModule, torch.fx.GraphModule])rh   _guardstracingr&   r-   r  r  r   fx_passes.joint_graphr  rz  ri   CodeGen_codegenr  r  )rS   r;   r  r  r  r'  s        @r=   r   r     s     *.B@)@3@@LO@	:@ 
		t	$++.0+/	
 	 
% I24BHH5" ..0BHHHH  "LLNI/ 
%	$s   C;;
D	c                    / n[         R                  R                  U R                  U R                  4UR
                  5        U$ r7   )rh   ri   r   r;   r<   r{   )r   r;   s     r=   r  r    s1    )+D	HHaffahh'5KrA   c                   [        [        U R                  5      5      n[        [        R
                  R                     " 5       n[        [         5      nS nU(       a  UR                  5       n[        U5       Vs/ s H  ofU;  d  M
  UPM     nnU(       a  X7S      R                  U5        OdUR                  U5        U(       a   UR                  ULa  UR                  U5        UnUR                  [        UR                  US5      5      5        U(       a  M  U(       d"  [        U5      [        U R                  5      :X  d   eg s  snf )Nr  r8   )r   r   r   r"   rh   ri   r   r   rK  r  r{   r  r   r   r   )r   pendingreadywaitingcursorr   r  waiting_fors           r=   stable_topological_sortr    s     8EKK()G uxx}}%'E $G F
{{}"'+@+Q%q+@ O$++D1IIdO&++T1d#F NN8GKKb$9:; '  3u:U[[)99999 As   :	EEc                p   ^  [         R                  [         R                  " T 5      SU 4S jj5       5       nU$ )z0Wrapper around lazy init functions in fx_passes/c                 h  > [         [           R                  5       n [        R                  R                  S 5         [        5          [        5          T" 5       nS S S 5        S S S 5        S S S 5        U [         [        '   W$ ! , (       d  f       N-= f! , (       d  f       N6= f! , (       d  f       N?= fr7   )r   r  rz   rh   r  r  r   r*   )counters_refr  rS   s     r=   	lazy_init%init_once_fakemode.<locals>.lazy_init  ss      (--/]]""4(*@*BNDTTF EU*B( ) EUDT*B*B((s;   B#BBB"B#
BB
B 	B##
B1)rC   r   )r   rl  r  )rS   r  s   ` r=   init_once_fakemoder     s4     ____R	  	 rA   c                   ^  SU 4S jjnU$ )z2Function for extra_check to put pass behind a flagc                $   > [        [        T5      $ r7   )r  r%   )r$  r`  s    r=   
flag_checkconfig_flag.<locals>.flag_check	  s    vt$$rA   )r$  r   rC   r   r8   )r`  r  s   ` r=   config_flagr  		  s    % rA   c                L     " S S[         5      nU" U 5      R                  5       $ )Nc                  ,   ^  \ rS rSrSU 4S jjrSrU =r$ )clone_graph.<locals>.CopyGraphi	  c                T  > [         TU ]  U5      n[        U[        R                  R
                  5      (       an  UR                  R                  R                  UR                  5        U R                  R                  R                  UR                  S 5      UR                  l        U$ r7   )r   r  r   rh   ri   Proxyr   r~   r}   	new_graph_graph_namespacerf  r`  )r:   r   r   r   s      r=   r  'clone_graph.<locals>.CopyGraph.run_node	  st    w'1H(EHHNN33""))(--8%)^^%D%D%P%PMM4&" OrA   r8   )r   r   rC   r   )r5   rD   rE   rF   r  rH   r  r  s   @r=   	CopyGraphr  	  s    	 	rA   r  )r+   	transform)input_graphr  s     r=   r  r  	  s$    	K 	 [!++--rA   _seen_patternsc                    [        U R                  5      U:  a  U R                  U   $ Uc  g U R                  R                  U5      $ r7   )r   r;   r<   ry   )r   
arg_number
kwarg_names      r=   get_arg_valuer  %	  s?     499~
"yy$$		{{z**rA   c           	     (   U/n[        U[        R                  R                  5      (       a9  UR	                  UR                  5        Vs/ s H  n[        X5      PM     sn5        U  Vs/ s H  oDR                  U;   d  M  UPM     sn$ s  snf s  snf r7   )r   rh   r  r  r   r  r  r   )r   rS   r  r  r   s        r=   filter_nodesr  0	  sm    $C"ejj1122

",,.I.hGB).IJ"9UTkkS&8DU99 J9s   
B
*BBc                    U R                   S:X  aU  [        U R                  [        5      (       d   e[	        U R
                  R                  U R                  5      R                  $ U R                  $ )zFor call_function and call_method, we directly use the target function;
For call_module, the target is string, and we treat the module class
 as a function.
r  )r  r   r   r4   r   r   r  r   r  s    r=   r  r  8	  sR    
 ww-$++s++++114;;?III;;rA   )r  )r   r   r   r   r   r4   rC   ra   )r  r+  rC   zTypeIs[Match]r  )r  r  r4  r  rC   ra   r'  )
r   r   r   zOptional[torch.fx.Graph]r  zdict[str, list[Optional[str]]]r  r   rC   r   )r  r2   r  rJ   r%  Iterable[Any]r   rP   r  r  rw  rv  r  )Union[dict[str, Union[float, int]], None]r  Sequence[str]r  zUnion[PatternExpr, None]r  r   rC   r   )rC  r4   r  r2   r%  r  r   rP   r  r  rC   r   )rC  r4   r  r2   r  rJ   r%  r  r   rP   r  r  rw  rv  r  r  r  r  r  r   rC   ra   )Nr8   )r  r2   r%  r  r   rP   r  r  r  r  rC   z(tuple[PatternExpr, torch.fx.GraphModule])r  r2   r%  r  r   rP   r  r  r  r  rC   r   )
r   r   rw  rv  rb  rn  r~  r   rC   z2Callable[[Callable[..., Any]], Callable[..., Any]])r   r  r   r   rC   r   )r  ztorch._ops.OpOverloadrC   r   r  )r  r   r  r   rC   r   )r   r  r   r   rC   rV  )r   r  rC   r   )r   r  rC   ra   )rS   r   r  z	list[str]rC   r   )r;   r   r<   r   rC   r   )r8   r8   Nr8   )r'  r  rU  zSequence[type[Any]]r  r  r  r  r  r  rC   r   )
rS   r  r;   r  r   r   r  zOptional[Callable[..., Any]]rC   rU   )rS   r  r;   r  rC   rU   )r   r   rC   zlist[torch.fx.node.Argument])rS   r  rC   zCallable[[], Any])r`  r4   rC   zCallable[[Match], Any])r  rU   rC   rU   r7   )r   r   r  rV  r  r  rC   r   )r   zIterable[torch.fx.Node]rS   r   rC   r   )r   r   rC   ztorch.fx.node.Target)r  
__future__r   r   dataclassesr   rN  r  r  loggingr  r  r  r.  rl  abcr   r   collectionsr   collections.abcr   r   r	   r
   r   r   pathlibr   r   r   r   r   r   r   typing_extensionsr   r   rh   torch._guardstorch.fxtorch.utils._pytreer   r   r  torch._dispatch.pythonr   torch._dynamo.utilsr   torch._prims_commonr   torch._subclasses.fake_tensorr   "torch.fx.experimental.proxy_tensorr   %torch.fx.experimental.symbolic_shapesr   r   torch.fx.graph_moduler   torch.fx.immutable_collectionsr   r   (torch.fx.passes.graph_transform_observerr    rv   r!   torch.utils._ordered_setr"   
_functorchr%   r"  _functorch.aot_autogradr&   r'   _functorch.partitionersr(   _subclassesr)   r*   ri   r+   r  decompositionr-   loweringr.   	getLoggerr5   r  r   atenprimsConstantr   r*  r  ry   r  r2   rJ   rP   rV   r   Targetr4   FnsTyperX   r[   r   r   r  r   r+  r  r   r   rH  rO  r\  rt  r~  r   _SimpleSpecr  r  r  r  r
  r  r  r  r  r)  rD  rZ  rn  	dataclassru  r  r  r   r  r  r  r)  r*  rG   rH  __file__parentr=  rK  rS  r#  r$  r@  rg  rk  rn  compiler{  rs  r}  r  r  r  r  r  r  r  r  no_gradr   enable_gradr   r  r  r   r  r  r  r  r  r  r8   rA   r=   <module>rD     s  !F #         	 	   # # X X  D D *    $ $ ; ( 0 @ 6 W + I K 1 / 3 C 7 4   . ; !yy~~		x./
**..>

K>x >> >#h # CL $$c)
*? ? : IK??(5?BE?	?8C CL, 8 E;&''
 '
T 1#  1F-+ -k E E(E+ E4O
+ O
d CHoo
k o
d?    (, * * ,
[ ,
^F
 F
R%
; %
P4 4n!X ! : : :: 
< 
 
 	= 	= 	= g
l g
 g
T "	''#' 2' 	'
 
'` ,8CG)+26!RRR "R 	R
 @R )R AR 'R 0R R 
Rj )3 o 4MMM "M 	M
 AM M` x.//+=@UU     $ ,8CG)+!555 5 "	5
 5 @5 )5 A5 '5 5 
5p e4
 DH)+!  A	
 ' . 5L DH)+				!		 		 A			
 '		 		 ,8
 ( 	
  84 ,8
 ( 	
  8,+ **VW	.(H
D; X Xv )+ CG)+k3k%k k A	k
 'k k\ 
 #'26
  	
 0  B # #L!:H&.  #-, . GK+
+%(+6C++:rA   