
    ȅiIE                        S r SSKrSSKrSSKJr  SSKJrJrJr  SSK	r	SSK
Jr  SSKJr  SSKJr  SSKJr  S	S
KJrJr  S	SKJrJrJrJrJrJr  S	SKJr  SSKJr  SSK J!r!  SSK"J#r#  SSK$J%r%  SSK&J'r'  SSK(J)r)  \(       a  SSK*J+r+   " S S\,5      r- " S S\,5      r.\" \/S5      r0S\	Rb                  S\24S jr3 " S S\)5      r4g)a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
    N)Iterable)AnyOptionalTYPE_CHECKING)TensorVariable)Source)getArtifactLogger)tree_map_only   )GuardBuilderinstall_guard)
AttrSourceConstDictKeySourceDictGetItemSourceGetItemSourceGlobalWeakRefSource
GradSource)GLOBAL_KEY_PREFIX   )VariableTracker)ConstantVariable)ConstDictVariable)ListVariable)GetAttrVariable)UserDefinedObjectVariable)InstructionTranslatorc                       \ rS rSrSrg)ArgMappingException8    N__name__
__module____qualname____firstlineno____static_attributes__r        [/home/james-whalen/.local/lib/python3.13/site-packages/torch/_dynamo/variables/optimizer.pyr   r   8       r'   r   c                       \ rS rSrSrg)GuardInstallException<   r    Nr!   r    r'   r(   r+   r+   <   r)   r'   r+   
perf_hintsxreturnc                 :   SSK Jn  U R                  (       a  U" U R                  R                  S5      n[
        R                  R                  R                  U 5      S LnU(       a3  UR                  c   eU=(       d    UR                  R                  U 5      $ U$ g)Nr   )get_managerFT)torch._inductor.cudagraph_treesr1   is_cudadeviceindextorch_dynamoutilsget_static_address_typecurrent_node_is_cuda_graph_recorded_tensor)r.   r1   manageris_static_addresss       r(   _is_static_for_cudagraphsr>   C   s    ;yyahhnne4!MM//GGJRVV''333! J''FFqI
 %$ r'   c                     ^  \ rS rSrSSS1\R
                  kr   SS\R                  R                  S\	\
\\4      S\	\\      S\	\
\R                  \4      S\SS4U 4S	 jjjrS
SS\S\\   S\
\\4   SS4
U 4S jjrS
SS\S\4U 4S jjrSS jrSS jrS\S\S\\\   \
\\4   4   4S jrSS jrSS jrS
SS\R                  S\4S jrS
SS\\   S\S\\   S\SS4S jrSS jr Sr!U =r"$ ) OptimizerVariableV   grad_to_sourcetensor_to_sourcestatic_tensor_namesNvaluekwargsr/   c                    > [         TU ]  " U40 UD6  Xl        U=(       d    0 U l        U=(       d    0 U l        U=(       d
    [        5       U l        g N)super__init__rE   rB   rC   setrD   )selfrE   rB   rD   rC   rF   	__class__s         r(   rJ   OptimizerVariable.__init__^   sE     	)&),1
,2 0 6B#6#?#% r'   txr   nameargsr   c                 `  > US:X  a  [        U R                  S5      (       d  [        T
U ]  XX45      $  U R	                  U5        U R                  5         U R                  " U0 UD6u  pVU R                  R                  " U0 UD6nU R                  U5        U R                  XXEU5        S[        U R                  5       3nUR                  XR                  5        U R                  U5        [        R                  " U5      $ [        T
U ]  XX45      $ ! [        [         4 a
  n	 Sn	A	N%Sn	A	ff = f)zVThis is an optimization to avoid tracing the very slow initialization of the optimizer_init_group__optimizer_N)hasattrrE   rI   call_methodgraph_break_if_pending_mutationmove_step_if_cpuget_python_argsrS   map_sources_and_install_guardsupdate_list_argsidstore_global_weakref_by_idcreate_finalizerr   creater   r+   )rL   rO   rP   rQ   rF   py_args	py_kwargsret_valmangled_name_rM   s             r(   rV   OptimizerVariable.call_methodm   s    = 4::}55w*2TBB44R8%%'%)%9%94%J6%J"**00'GYG33B7%%bK ".bn-=>--lJJG%%b) (..w77
 w"2T::	 ()>? s   CD D-(D-c                 *  > US;   a1  U R                   (       d   e[        X[        U R                   U5      S9$ US:X  aG  SSKJn  U R
                  R                   H  nUS    H
  nU" USS9  M     M     U R                  U5        [        TU ]%  X5      $ )	NrS   )sourceparam_groupsr   mark_static_addressparamsTguard)
rg   r   r   
decoratorsrj   rE   rh   _set_capturablerI   var_getattr)rL   rO   rP   rj   groupprM   s         r(   rp   OptimizerVariable.var_getattr   s     M";;;"4jd6STT>!800xA'6 ) 1   $w"2,,r'   c           	      8   U R                   R                   H  nUS    Ht  nUR                  R                  nUR                  R                  [        U5      S 5      nU(       d  MG  UR                  U5      (       d  M_  SSKJ	n  U" SSU SU 3S/ S9  Mv     M     g )	Nrk   r   )unimplementedz(optimizer: pending mutation on parameterz
variable: z, parameter: zSPending mutations on a parameter (e.g. due to using closure) require a graph break.)gb_typecontextexplanationhints)
rE   rh   outputside_effectsid_to_variablegetr\   has_pending_mutationexcru   )rL   rO   grr   r{   variableru   s          r(   rW   1OptimizerVariable.graph_break_if_pending_mutation   s    
 ((Ax[!yy55'66::2a5$G8 A A( K K3! J",XJmA3 G$y 	 ! )r'   c                 >  ^  SSK Jn  S[        [        [        4   S[
        4U 4S jjnT R                  R                   H  nU" U5      (       d  M  SUS'   M     T R                  =(       a    [        T R                  S5      nUR                  [        R                  " UT R                  R                  U5      5      nUR                   HP  n[        R                  " [         R"                  " S5      5      n[         R"                  " S5      UR                  U'   MR     g )	Nr   LazyVariableTrackerrq   r/   c                    > SnSnU R                  S/ 5       H?  nX#R                  =(       d    UR                  -  nXTR                  R                  ;  -  nMA     SU ;   =(       a    U=(       a    U$ )NTrk   
capturable)r}   r3   is_xpurE   state)rq   all_uninitializedall_gpurr   rL   s       r(   safe_to_set_capturableAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturable   sh     $GYYx,9900!djj.>.>%>>! -  5(J->J7Jr'   Tr   rh   ) r   dictstrr   boolrE   rh   rg   r   realize_allr   builditemsr   _HashableTrackerr   r_   )	rL   rO   r   r   rq   rg   param_groups_vtparam_group_vtkeys	   `        r(   ro   !OptimizerVariable._set_capturable   s    )	K$sCx. 	KT 	K ZZ,,E%e,,&*l# - HDKK!H-99!!"djj&=&=vF
 .33N#44 ''5C )9(?(?(EN  %	 4r'   c                    ^  S[         S[         4U 4S jjnU Vs/ s H
  oC" U5      PM     nnUR                  5        VVs0 s H  u  pgXc" U5      _M     nnnXX4$ s  snf s  snnf )z9Get python values equivalent to the variable tracker argsargr/   c                 0  > [        U [        5      (       a%  U R                  5       (       a  U R                  5       $ [        U [        5      (       a  U R
                  (       d  / $ [        U [        5      (       a  [        U R                  [        5      (       az  [        U R                  R                  [        5      (       aQ  U R                  R                  R                  S:X  a-  TR                  R                  U R                  R                     $ [        e)Nrh   )
isinstancer   is_python_constantas_python_constantr   r   r   rg   r   baser   memberrE   rh   r5   r   )r   rL   s    r(   map_arg2OptimizerVariable.get_python_args.<locals>.map_arg   s    #//C4J4J4L4L--//C..syy	3 122szz=99szz
;;JJOO**n<zz..szz/?/?@@%%r'   )r   r   )	rL   rQ   rF   r   r   new_argskv
new_kwargss	   `        r(   rY   !OptimizerVariable.get_python_args   se    
	& 	& 	& -11DSGCLD106?am
?## 2?s   A Ac                     U R                   R                  R                  5        HD  u  pSU;   d  M  US   R                  (       d  M#  US   R	                  UR
                  5      US'   MF     g )Nstep)rE   r   r   is_cputor4   )rL   rr   r   s      r(   rX   "OptimizerVariable.move_step_if_cpu   sS    

((..0HA5=#7#7#7 %f 0 0 :f 1r'   c                 R	  ^ SSK Jm  SSKJn  0 U l        0 U l        S[        SS 4U4S jjn[        [        R                  X0R                  R                  5        U R                  =(       a    [        U R                  S5      nUR                  [        R                   " XR                  R"                  U5      5      nU R                  =(       a    [        U R                  S	5      n[        R                   " XR                  R                  U5      nUR%                  5         Uc   eUR&                  R(                  R+                  U5        [-        U R                  R"                  UR.                  5       GH,  u  p[1        US
   5      S:  a  US
    H  n
U
R2                  c  M  S n[5        U R                  R                  R7                  5       5       H  u  pXL d  M  Un  O   U(       d  M[  UR                  [        R                   " UU R                  R                  U
   [9        U[;        Xk5      5      5      5          O   U	R=                  U[>        R@                  " S
5      5      nSn/ n[-        US
   URC                  U5      5       H  u  nnUR                  nUU R
                  U'   [E        US5      nUR2                  bJ  UU R                  UR2                  '   [G        UR2                  5      (       d  SnURI                  U5        M  M  [K        URM                  [N        RP                  5      5        M     U(       a  GM  [R        RU                  [V        RX                  5      (       d  GM  U Vs/ s H  nURZ                  PM     nn[R        R]                  SU5        GM/     [5        U R                  R                  R_                  5       5       H  u  nn[9        U[;        UU5      5      nUR&                  R(                  R+                  U5        [5        UR_                  5       5       Hn  u  nn[a        U[        R                  5      (       d  M'  UU R                  ;  d  M9  UU R
                  ;  d  MK  [9        U[;        UU5      5      U R
                  U'   Mp     M     g s  snf )Nr   ri   r   r   r.   r/   c                    > T" U SS9  g )NTrl   r    )r.   rj   s    r(   mark_staticEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_static  s    .r'   rh   r   rk   r   TgradF)zGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.)1rn   rj   lazyr   rB   rC   r   r
   r6   TensorrE   r   rg   r   r   r   r   rh   realizerz   guard_on_key_orderaddzipr   lenr   	enumeratekeysr   r   getitem_constr   r_   unpack_var_sequencer   r>   appendr   
make_guardr   CONSTANT_MATCHperf_hint_logisEnabledForloggingDEBUGrP   warningvaluesr   )rL   rO   r   r   params_groups_sourcer   state_sourcestate_vtrq   group_vtparam	key_indexir   	params_vt
all_staticnon_static_gradsrr   p_vtparam_sourcegrad_sourcesrcnon_static_grad_namesidxrE   p_state_source	inner_idxr   rj   s                               @r(   rZ   0OptimizerVariable.map_sources_and_install_guards   s   4-  "	/3 	/4 	/ 	ellK1A1AB  ${{Vz$++~/V-99!!"jj&=&=?ST
 {{Gz$++w'G"((ZZ-=-=|L 	'''
		$$((6  #4::#:#:O<Q<QROE 5?#a'"8_Ezz-$(	$-djj.>.>.C.C.E$FDA z,-	 % %G %9/;; / 5 5$&$(JJ$4$4U$;$5(4(:<(S%&!"	 "% -( !..r3C3J3J83TUIJ!uX	0M0Mb0QR4#{{+7%%a(( 
 66%2=D''/4QVV<<%*
(//< = "+"8"89T9T"UV S$ :-"<"<W]]"K"K=M(N=Mc=M%(N%%
 *_  St $DJJ$4$4$;$;$=>JC.0sCN II((,,^< )%,,. 9	1q%,,//!4!44!6!66/@&(:>9(U0D))!, !: ? )Os   3R$tensor_valuec                    SSK Jn  X R                  ;   aU  U" USS9  U R                  U   nU R                  R	                  UR
                  R                  UR                  5      5        OX R                  ;   a  U R                  U   nOfU" USS9  UR                  [        U5      n[        U5      nU R                  R	                  UR
                  R                  UR                  5      5        [        R                  " XU5      $ )z%Wrap state tensor in a TensorVariabler   ri   Trl   )rn   rj   rC   rD   r   rz   module_key_namerP   rB   r]   r   r   r   r   )rL   rO   r   rj   rg   global_names         r(   wrap_tensorOptimizerVariable.wrap_tensora  s     	5 000D9**<8F$$(()B)B6;;)OP000((6F  D9778I<XK(5F$$(()B)B6;;)OP$$Rv>>r'   r`   ra   c           	      P   [        X$5       GH  u  pg[        U[        5      (       d  M  [        U[        5      (       d   S5       e[	        U5       H  u  pUR
                  R                  R                  U5        [        U	[        R                  5      (       a,  UR                  R                  U R                  X5      5        Mu  UR                  =(       a    [        UR                  U5      n
UR                  R                  [        R                   " XU
5      5        M     GM     g)z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)r   r   r   listr   rz   r{   mutationr6   r   r   r   r   rg   r   r   r   )rL   rO   rQ   rF   r`   ra   r   py_argr   valrg   s              r(   r[   "OptimizerVariable.update_list_args}  s     t-KC#|,,!&$// C/ (/FAII**33C8!#u||44		(()9)9")BC!$!Lcjj!0L		(()>)>r)OP 0 .r'   c                    ^^^ U R                   mU R                  mUR                  R                  mS[        R
                  R                  SS 4UUU4S jjnUR                  R                  U5        g )Ngmr/   c                 J   >^  SU UU4S jjn[         R                  " TU5        g )Nc                  8  > T H  n TR                   R                  U S 5        TR                  R                  U S 5        TR                  (       a  TR                  R	                  5         TR
                  (       d  My  TR
                  R	                  5         M     g rH   )_bufferspop_parametersparams_flatclearparams_flat_unwrap_subclasses)rP   r   names_to_deletetcs    r(   clear_static_tensor_refs\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refs  sh    +DKKOOD$/NN&&tT2~~,,.77788>>@ ,r'   r/   N)weakreffinalize)r   r   r   r   rE   s   ` r(   init_finalizer:OptimizerVariable.create_finalizer.<locals>.init_finalizer  s"    A A U$<=r'   )rD   rE   rz   tracing_contextr6   fxGraphModuleadd_graph_finalizer)rL   rO   r   r   r   rE   s      @@@r(   r^   "OptimizerVariable.create_finalizer  s\    22

YY&&
	>uxx33 
	> 
	> 
	> 			%%n5r'   )rB   rD   rC   rE   )NNN)rO   r   r/   Nr   )#r"   r#   r$   r%   r   _nonvar_fieldsr6   optim	Optimizerr   r   r   r   rK   r   r   r   rJ   r   r   rV   rp   rW   ro   tuplerY   rX   rZ   r   r   r   r[   r^   r&   __classcell__)rM   s   @r(   r@   r@   V   s    
#	1	1	N ;?26AE@{{$$@ !c:o!67@ &c#h/	@
 #4f(<#=>@ @ 
@ @";#"; "; ?#	";
 S/)*"; 
";H-5 -S -_ -&&F@$$$'$	tCy$sCx.(	)$<;
eN?)?9>?	?8Q#Q 'Q 	Q
 #Q Q 
Q,6 6r'   r@   )5__doc__r   r   collections.abcr   typingr   r   r   r6   torch._dynamo.variables.tensorr   torch._guardsr   torch._loggingr	   torch.utils._pytreer
   guardsr   r   rg   r   r   r   r   r   r   r8   r   r   r   constantr   dictsr   listsr   miscr   user_definedr   torch._dynamo.symbolic_convertr   	Exceptionr   r+   r"   r   r   r   r>   r@   r    r'   r(   <module>r     s   ,   $ / /  9   , - 0  & ! & $  ! 3 D	) 		I 	 "(L9 $ &N61 N6r'   