
    ȅi<C                        S r SSKrSSKJr  SSKJr  SSKJrJrJ	r	J
r
JrJrJr  SSKJr   " S S\5      r   S"S	\\\	   -  S
\S\S\S-  S\S\S\\\S-        4S jjr " S S5      r\R0                  \" SS5      \R2                  \" SS5      \R4                  \" SS5      \R6                  \" SSS5      \R8                  \" SSS5      0rS	\\\S-        S\\\S-        4S jr S#S	\\\S-        S \S-  SS4S! jjrg)$z
This visualizer requires matplotlib to be installed.

Example usage:

ops = get_schedule_ops("InterleavedZeroBubble", 4, 8)
visualize_schedule(ops, "test.png")
    N)
NamedTuple)mock)_Action_ComputationType_PipelineSchedule_PipelineScheduleRuntimeget_schedule_classPipelineScheduleMultiPipelineScheduleSingle)PipelineStagec                   4    \ rS rSr% \\S'   \\S'   \\S'   Srg)OpKey   stage_indexcomputation_typemicrobatch_index N)__name__
__module____qualname____firstlineno__int__annotations__r   __static_attributes__r       k/home/james-whalen/.local/lib/python3.13/site-packages/torch/distributed/pipelining/_schedule_visualizer.pyr   r      s    &&r   r   schedule	pp_degreenum_microbatchesnum_stages_per_rankadd_spacing
with_commsreturnc                 l   U(       a  U(       a  [        S5      e[        U [        5      (       a  [        U 5      nO&[	        U [
        5      (       a  U nO[        SU  35      e[        R                  " [        SS9nSUl	        Xl
        SUl        [	        U[        5      (       a%  Uc  SnUS:X  d  [        SU 35      eUnX1-  Ul        Oi[	        U[        5      (       aF  Uc  S	nUS	:  d  [        S
U 35      e[!        U5       V	s/ s H  oPM     nn	U H  n
X1-  U
l        M     O[        SU 35      eU" X5      nUR"                  c  [        S5      e/ nU(       a`  [%        X5      nUR'                  UR"                  5        [!        U5       H*  nUR)                  [+        UR,                  U   5      5        M,     O0[!        U5       H!  nUR)                  UR"                  U   5        M#     U(       a3  U VVs/ s H  o Vs/ s H	  oc  M  UPM     snPM     nnn[/        U5      nU$ s  sn	f s  snf s  snnf )aK  
Get all actions for a given schedule, pp_degree, and num_microbatches. The actions are returned in a list of lists
where each inner list represents a rank and each element in the inner list represents an action.

The schedule can be specified as a string which is passed into get_schedule_class() or a _PipelineSchedule instance.
z2Cannot add spacing and view comms at the same timezInvalid schedule: T)instancer   N   z*Expected num_stages_per_rank to be 1, got    z'Expected num_stages_per_rank >= 2, got z&Expected pipeline_order to not be None)
ValueError
isinstancestrr	   
issubclassr   r   create_autospecr   
group_rank
group_sizesubmodr   AssertionError
num_stagesr
   rangepipeline_orderr   _prepare_schedule_with_commsappendlistpipeline_order_with_commsadd_schedule_op_spacing)r   r   r   r    r!   r"   schedule_classmock_pipeline_stagestages_stageschedule_instanceall_actionsruntimerankactions                   r   get_schedule_opsrC   "   sR    zMNN(C  +H5	H/	0	0!-hZ899 ..}tL%&"%."!% ."899&"#"a' <=P<QR  %/;	N$9	:	:&"#"a' 9:M9NO  055H/IJ/I!%/IJE2>E  -n-=>?? 'v@''/EFF /1K*6D,,->-M-MN)$DtG$E$Ed$KLM % )$D0??EF %  KV
JU$$=$V$=+ 	 
 .k: E K: >
s$   H&6	H0?H+	H+H0+H0c                   2    \ rS rSr  SS\S\S\4S jjrSrg)	_ComputationTypeVisualw   colortextwidthc                 (    Xl         X0l        X l        g N)rG   rI   rH   )selfrG   rH   rI   s       r   __init___ComputationTypeVisual.__init__x   s     

	r   )rG   rH   rI   N) r&   )r   r   r   r   r*   r   rM   r   r   r   r   rE   rE   w   s0     	  	 r   rE   blueForwardtealzBackward InputgreenzBackward WeightorangezFull Backwardr'   purplezOverlap F+B   c                   ^^^^^^ U (       d  U $ [        S U  5       5      S-   m[        U 5      n[        U5       Vs/ s H  n/ PM     snmU  Vs/ s H  n[        R                  " U5      PM     nn0 mS[
        S[        S[        4U4S jjmS[        S[        [
           4U4S jjmS[        S[        S[        4UUU4S	 jjmS[        S
[        S[        S[        4UU4S jjnSnSn[        R                  [        U5      S5      nU(       Ga8  [        SU 35        [        U5       H  u  pU
(       d  M  XI   n
U
S   n[        SU	 SU< 35        Uc(  TU	   R                  S5        U
R                  5         SnMT  XhU	   :  d  M^  T" X5      (       d  Mm  U" XU5      X'   U
R                  5         SnM     [        U5       H!  n	XhU	   :  d  M  TU	   R                  S5        M#     U V
s/ s H  o(       d  M  U
PM     nn
US-  nUS-  nU[        S [         R#                  5        5       5      :  a  [%        S5      eU(       a  GM8  T$ s  snf s  snf s  sn
f )ah  
Add spacing to the schedule based on dependencies between ranks.

Before adding an operation to the list, this function checks if there are
dependencies from other ranks. If there are dependencies (other ranks have
not finished processing the required microbatch), it adds None instead.

For example, Forward microbatch 0 on rank 1 depends on rank 0 processing
Forward microbatch 0 first.

Args:
    schedule: The original schedule as a list of lists where each inner list
             represents a rank and each element represents an action.

Returns:
    A new schedule with proper spacing based on dependencies.
c              3   X   #    U  H   nU  H  nUc  M  UR                   v   M     M"     g 7frK   )r   ).0rank_actionsrB   s      r   	<genexpr>*add_schedule_op_spacing.<locals>.<genexpr>   s/      
 (& F&  (s   **r&   dependency_keytimestepr#   c                 ,   > U T;   =(       a    UTU    :  $ )zDCheck if a dependency operation has completed by the given timestep.r   )r]   r^   scheduled_opss     r   is_dependency_ready4add_schedule_op_spacing.<locals>.is_dependency_ready   s$     m+ :M.99	
r   rB   c                   > U R                   nU R                  nU R                  nUc  [        SU  S35      eUS:X  a  U[        R
                  :X  a  / $ UTS-
  :X  aC  U[        R                  [        R                  4;   a  [        US-
  [        R
                  U5      /$ U[        R
                  :X  a  [        US-
  [        R
                  U5      /$ U[        R                  [        R                  4;   a<  [        US-   [        R                  U5      [        US-   [        R                  U5      /$ U[        R                  :X  a  [        U[        R                  U5      /$ [        SU 35      e)z+Get the list of dependencies for an action.Action  has None microbatch_indexr   r&   Unknown computation type: )r   r   r   r0   r   FORWARDFULL_BACKWARDBACKWARD_INPUTr   BACKWARD_WEIGHTRuntimeError)rB   	stage_idx	comp_typemb_idxr1   s       r   get_dependencies1add_schedule_op_spacing.<locals>.get_dependencies   sm   &&	++	(( > 76(2L!MNN >i+;+C+CCI 
Q&9**++9
 ,
 )a-)9)A)A6JKK (000)a-)9)A)A6JKK **++
 

 i!m%5%C%CVLi!m%5%D%DfM  (888)%5%D%DfMNN7	{CDDr   c                 H  >^ U R                   [        R                  [        R                  [        R                  4;   a  T" U 5      n[        UU4S jU 5       5      $ U R                   [        R                  :X  a  T" U 5      n[        UU4S jU 5       5      $ U R                   [        R                  :X  aT  U R                  c  [        SU  S35      e/ nU R                   H  nUR                  T" UT5      5        M     [        U5      $ [        SU R                    35      e)zBCheck if an action is ready to be scheduled at the given timestep.c              3   6   >#    U  H  nT" UT5      v   M     g 7frK   r   rY   depra   r^   s     r   r[   Cadd_schedule_op_spacing.<locals>.is_action_ready.<locals>.<genexpr>        R\c*399\   c              3   6   >#    U  H  nT" UT5      v   M     g 7frK   r   rs   s     r   r[   ru      rv   rw   OVERLAP_F_B action  has None sub_actionsrf   )r   r   rh   ri   rj   anyrg   allOVERLAP_F_Bsub_actionsr0   r5   rk   )rB   r^   dependenciesdep_list
sub_actionro   is_action_readyra   s    `   r   r   0add_schedule_op_spacing.<locals>.is_action_ready   s	    ""**++,,'
 

 ,F3LR\RRR$$(8(@(@@+F3LR\RRR$$(8(D(DD!!)$)&1FG  $&H$00

H EF 1x= !;F<S<S;TUVVr   rA   c                 \  > T
U   R                  U 5        U R                  n[        U   R                  nX$-   nU[        R
                  :X  a  U R                  c  [        SU  S35      eSnU R                   Hp  nUR                  c  [        SU S35      e[        UR                     R                  nXh-  nX&-   T	[        UR                  UR                  UR                  5      '   Mr     U$ U R                  c  [        SU  S35      eUT	[        U R                  X0R                  5      '   U$ )z.Schedule an action and return completion time.ry   rz   r   zSub-action re   rd   )r5   r   action_type_to_color_mappingrI   r   r}   r~   r0   r   r   r   )rB   rA   r^   rm   	comp_timecompletion_timecumulative_timer   sub_comp_timer`   spaced_schedules            r   schedule_action0add_schedule_op_spacing.<locals>.schedule_action  s_   $$V,++	0;AA	".(444!!)$)&1FG   O$00
..6(%j\1KL  !=//!%   0 . ".."33"33 1.  &&.$wvh6P%QRR   f(()5L5LM r   r   zCurrent timestep: zRank: z	, action=Nc              3   8   #    U  H  oR                   v   M     g 7frK   )rI   )rY   visuals     r   r[   r\   O  s      ,
'LVLL'L   z2No progress made in scheduling - possible deadlock)maxlenr2   collectionsdequer   r   boolr   r6   dictfromkeysprint	enumerater5   popleftr   valuesrk   )r   	num_ranksr<   opsrank_opsr   current_timesteptimesteps_without_progressrank_completion_timesrA   op_queuerB   ro   r   ra   r1   r`   r   s               @@@@@@r   r8   r8      s   (  	 
 (
 	
 	  HI?DY?O2P?O!2?O2PO2:;(3!!#&(H; ')M
E 
S 
T 
'E 'ET%[ 'ERW W3 W4 W W4% %s %c %c % %P !" MM%	*:A>
"#3"456'1ND~Ha[FF4&
6),-~%,,T2  "-.*!4%@@_F F />"2/%+   "-.*% 2* )$D#>>%,,T2 %
 .6BXHXBA"a'"% ,
'C'J'J'L,
 )
 
 STTI (L C 3Q;n Cs   I I,
I:Ifilenamec                 *   SSK Jn  SSKJn  SUR                  S'   [        U 5      n[        S U  5       5      nUR                  US-   US-   4S9u  pgS	n[        XT5      S
-   n	[        5       n
[        U 5       H  u  pSnU H  nUb  [        R                  UR                  [        S5      5      nU
R                  UR                  5        UR                   nUR"                  nUR$                  b  SnSnOSnSnU" XU-
  S-
  4USUSUS9nUR'                  U5        UR)                  UUS-  -   XK-
  S-
  S-   [+        U5      SSU	SUS9  UU-  nOUS-  n[        X5      nM     M     UR-                  SUS-   5        UR/                  SUS-   5        UR1                  [3        U5       Vs/ s H
  oU-
  S-
  PM     sn5        UR5                  [3        U5       Vs/ s H  nSU 3PM
     snU	S9  UR7                  / 5        UR9                  S5        U
 Vs/ s H0  nU" SSS[        U   R                   S[        U   R(                  S9PM2     nnUR;                  USU	S9  U(       a  UR=                  USS9  gUR?                  5         gs  snf s  snf s  snf )aM  
Visualize the schedule using matplotlib.
The schedule is a list of lists where each inner list represents a rank and each element in the inner list represents an action.
The actions are represented as rectangles with different colors based on their computation type.
The filename is optional and if provided, the plot will be saved to that file.

Args:
    schedule: The schedule to visualize.
    filename: The filename to save the plot to. If not provided, the plot will be displayed.
    add_schedule_spacing: If True, add spacing to the schedule based on dependencies between ranks.

r   N)	RectanglezDejaVu Sanszfont.familyc              3   8   #    U  H  n[        U5      v   M     g 7frK   )r   )rY   rA   s     r   r[   %visualize_schedule.<locals>.<genexpr>o  s     5HDc$iiHr   r'   )figsize   blacknormalr&   )	facecolor	edgecolor	linewidthg      ?centerwhite)havafontsizerG   weightg      zRank )r   F)r   r   )r   r   labelzupper right)handleslocr   tight)bbox_inches) matplotlib.pyplotpyplotmatplotlib.patchesr   rcParamsr   r   subplotsminsetr   r   getr   rE   addrG   rI   r~   	add_patchrH   r*   set_xlimset_ylim
set_yticksr2   set_yticklabelsset_xticklabelsgridlegendsavefigshow)r   r   pltr   r   max_actionsfigaxmax_draw_position	font_sizeused_computationrank_idxactionsdraw_positionrB   comp_type_colorrG   rI   r   text_weightrectirm   legend_elementss                           r   visualize_scheduler   W  s   " $, 	 LL HI5H55K llK!OY]#ClDGCK+a/Iu&x0F!">"B"B++-CG-L# !$$V%<%<='--'-- %%1 !I"*K !I"*K !"$81$<=#%' T" !EAI-(1,s2K&!&  	 &" #$5 EW  1\ KK'!+,KKi#o&MMeI>NO>N(x'#->NOPU9-=>-=%s-=>Sr GGEN *
 *I 	29=CC.y9>>	
 *  
 IIo=9IMH'2
/ P>
s   (JJ7J)NFFrK   ) __doc__r   typingr   unittestr   &torch.distributed.pipelining.schedulesr   r   r   r   r	   r
   r   "torch.distributed.pipelining.stager   r   r*   typer   r   r6   rC   rE   rg   ri   rj   rh   r}   r   r8   r   r   r   r   <module>r      s        =J  '+RD*++RR R t	R
 R R 
$w~
Rj	 	 4VYG##%;FDT%U$$&<"' ""$:/1%   "8=RS"T
  C4$'(C	$w~
CP  h4$'(hDjh 
hr   