
    /hI                         S SK Jr  S SKrS SKJs  Jr  S SKJr  S SK	J
r
  S SKJr  S SKJrJrJr  S SKJr  S SKJrJr  S S	KJrJr  S
 rSS jrSS jrS rS rS rSS jr SS jr S r!S r"S r#\$S:X  a  \"" 5         gg)    )deepcopyN)AdamW)LambdaLR)
DataLoader)AcceleratorDataLoaderConfigurationGradientAccumulationPlugin)GradientState)RegressionDatasetRegressionModel)DistributedTypeset_seedc           	         [        U R                  5       UR                  5       5       H  u  pVUR                  (       d  M  U(       dV  [        R                  " UR
                  UR
                  40 UD6SL d%   SU SUR
                   SUR
                   S35       eMu  [        R                  " UR
                  UR
                  40 UD6SL a  M   SU SUR
                   SUR
                   S35       e   g )	NF7Gradients in sync when they should not be at iteration z:
model_a grad (z) == model_b grad ()T7Gradients not in sync when they should be at iteration z) != model_b grad ()zip
parametersrequires_gradtorchallclosegrad)model_amodel_bdid_step	iterationkwargsparam
grad_params          a/home/james-whalen/.local/lib/python3.13/site-packages/accelerate/test_utils/scripts/test_sync.pycheck_model_parametersr!      s     !3!3!5w7I7I7KL"">>%**jooHHEQ I)Tefkfpfpeq  rE  FP  FU  FU  EV  VW  XQ
 >>%**jooHHDP I)Tefkfpfpeq  rE  FP  FU  FU  EV  VW  XP M    c                     U R                  5         U " U5      n[        R                  " XRR                  UR                  5      5      nU(       d  XcR
                  -  nUR                  5         g UR                  U5        g N)trainFmse_losstodevicegradient_accumulation_stepsbackward)modelinputtargetacceleratordo_backwardoutputlosss          r    
step_modelr3   -   sW    	KKM5\F::fii67D777T"r"   c                    [        S5        [        5       n[        U5      n[        SS9n[	        USS9nUR                  U R                  5        U(       aF  [        UR                  5       SS9n[        UR                  5       SS9n[        US S	9n[        US
 S	9nU(       a  U R                  UWWU5      u  p7pOU R                  X55      u  p5U(       a  UWXUWW4$ X#U4$ )z3Returns everything needed to perform basic training*   P   length   
batch_sizeMbP?)paramslrc                     U S-  $ Ng? epochs    r    <lambda>$get_training_setup.<locals>.<lambda>C   s    eTkr"   )	lr_lambdac                     U S-  $ r@   rA   rB   s    r    rD   rE   D   s    eTkr"   )r   r   r   r   r   r(   r)   r   r   r   prepare)	r/   schedr,   	ddp_modeldset
dataloaderoptddp_opt	ddp_scheds	            r    get_training_setuprP   8   s    RLEIB'DDR0J	HH[ 5++-$7y335$?(ABW0IJ	4?4G4G	SZ\egq4r1	Iz + 3 3I J	sEy'9MMZ''r"   c                    [        U 5      u  pn[        [        U5      5      R                  5       u  pE[	        S5       GHs  nU R                  XE45      u  pxUR                  U R                  5      UR                  U R                  5      p[        XX5        US-  S:X  a'  U R                  U5         [        X$XP5        S S S 5        O[        X$XP5        [        XSU5        [        UR                  5       UR                  5       5       Hj  u  pU	R                  (       d  M  [        R                  " U	R                   U
R                   5      (       a  MJ   SU	R                    SU
R                    S35       e   [        R"                  " SU-   5        U[        R$                  " ['        U5      5         nGMv     g ! , (       d  f       N= f)	N      r   T7Gradients not in sync when they should be:
Model grad () != DDP grad (r   9  )rP   nextitervaluesrangegatherr(   r)   r3   no_syncr!   r   r   r   r   r   r   manual_seedrandpermlenr/   r,   rJ   rL   	ddp_input
ddp_targetr   r-   r.   r   	ddp_params              r    test_noop_syncrd   O   sr   #5k#B Ej j!1299;I1X	#**I+BC!3!34fii@R@R6Sv55q=A$$Y/9I 0/ yZE 	uyA #E$4$4$6	8L8L8N OE&&>>%**inn== J5::,Vefoftfteuuvw= !P 	$*+ennS^<=	5  0/s   1F77
G	c                 *   [        U 5      u  pn[        [        U5      5      R                  5       u  pE[	        S5       GH  nU R                  XE45      u  pxUR                  U R                  5      UR                  U R                  5      p[        XX5        US-  S:X  a'  U R                  U5         [        X$XP5        S S S 5        O[        X$XP5        [        UR                  5       UR                  5       5       H  u  pU	R                  (       d  M  US-  S:X  aR  [        R                  " U	R                  U
R                  5      SL d"   SU	R                   SU
R                   S35       eMs  [        R                  " U	R                  U
R                  5      SL a  M   S	U	R                   S
U
R                   S35       e   [        R                   " SU-   5        U[        R"                  " [%        U5      5         nGM     g ! , (       d  f       GN?= f)NrR   rS   r   F7Gradients in sync when they should not be:
Model grad () == DDP grad (r   TrT   rU   rV   )rP   rW   rX   rY   rZ   r[   r(   r)   r3   r\   r   r   r   r   r   r   r]   r^   r_   r`   s              r    test_distributed_syncrh   q   s   #5k#B Ej j!1299;I1X	#**I+BC!3!34fii@R@R6Sv55q=A$$Y/9I 0/ yZE !$E$4$4$6	8L8L8N OE&&1}!~~ejj)..AUJ NuzzlZijsjxjxiyyz{J
 ~~ejj)..ATI NuzzlZijsjxjxiyyz{I !P 	$*+ennS^<=	A  0/s   1H
H	c                 V   [        U 5      u  pn/ nSn[        U5       H  n[        [        U5      5      R	                  5       u  pxU R                  Xx45      u  pU	R                  U R                  5      U
R                  U R                  5      p[        XX5        U R                  U5         U" U5      n[        R                  " XR                  UR                  5      5      nUR                  U5        S S S 5        M     [        U5       GHv  nXF   nXeS-
  :  a  U R                  U5        [        UR                  5       UR                  5       5       Hh  u  pUR                   (       d  M  ["        R$                  " UR&                  UR&                  5      SL a  MH   SUR&                   SUR&                   S35       e   M  U R)                  U5         U R                  U5        S S S 5        [        UR                  5       UR                  5       5       Hh  u  pUR                   (       d  M  ["        R$                  " UR&                  UR&                  5      SL a  MH   SUR&                   S	UR&                   S35       e   GMy     g ! , (       d  f       GMy  = f! , (       d  f       N= f)
NrR      Frf   rg   r   TrT   rU   )rP   rZ   rW   rX   rY   r[   r(   r)   r3   r\   r&   r'   appendr+   r   r   r   r   r   r   trigger_sync_in_backward)r/   r,   rJ   rL   lossesnum_iterationsr   ra   rb   r-   r.   
ddp_outputr2   r   rc   s                  r    "test_distributed_sync_multiple_fwdrp      s;   #5k#B EjFN>*	 $T*%5 6 = = ?	 $**I+BC!3!34fii@R@R6Sv 	55   +"9-J::j--
8I8I*JKDMM$ ,+ +" >*	 ))  & %((8(8(:I<P<P<R$S **~~ejj)..AUJ NuzzlZijsjxjxiyyz{J	 %T 55i@$$T* A %((8(8(:I<P<P<R$S **~~ejj)..ATI NuzzlZijsjxjxiyyz{I	 %T- + ,+0 A@s   +A	JJ
J	
J(	c           
         [        SUS9n[        XS9n[        UUS9n[        U5      u  pgn[	        U5       GH  u  pU
R                  5       u  pUR                  X45      u  pUR                  UR                  5      UR                  UR                  5      p[        XmXS5        UR                  U5         [        X{X5        S S S 5        [        UR                  5       UR                  5       5       H  u  nnUR                  (       d  M  U	S-   S-  S:X  d  U	[        U5      S-
  :X  d  U(       aU  [        R                   " UR"                  UR"                  5      SL d%   S	U	 S
UR"                   SUR"                   S35       eM  [        R                   " UR"                  UR"                  5      SL a  M   SU	 S
UR"                   SUR"                   S35       e   [        R$                  " SU	-   5        U[        R&                  " [        U5      5         nGM     [(        R*                  " 5         g ! , (       d  f       GNw= f)NrS   	num_stepssync_each_batchsplit_batchesdispatch_batchesdataloader_configgradient_accumulation_pluginFrj   r   Tr   z:
Model grad (rU   r   r   rg   rV   )r	   r   r   rP   	enumeraterY   r[   r(   r)   r3   
accumulater   r   r   r_   r   r   r   r]   r^   r
   _reset_state)rv   rw   rt   rz   ry   r/   r,   rJ   rL   r   batchra   rb   r-   r.   r   rc   s                    r    test_gradient_accumulationr      s;   #=[j#k /mo+%AK
 $6k#B Ej%j1	 %	#**I+BC!3!34fii@R@R6Sv5e<##I.yZE / !$E$4$4$6	8L8L8N OE9&&Q!#q(i3z?Q;N.NSb~~ejj)..ATI Mi[Xghmhrhrgs  tC  DM  DR  DR  CS  ST  UI
 ~~ejj)..AUJ Mi[Xghmhrhrgs  tC  DM  DR  DR  CS  ST  UJ !P 	$*+ennS^<=	9 2:  + /.s   6H33
I	c           	      "   [        SUS9n[        XS9n[        UUS9n[        US5      u  pgppn[	        U	5       GH*  u  pUR                  5       u  nnUR                  UU45      u  nnUR                  UR                  5      UR                  UR                  5      nnUR                  5         U
R                  5         [        UUUUS5        UR                  5         US-   S-  S:X  d  US-   [        U	5      :X  aD  U (       a  UR                  5         O,[        UR                  5       H  nUR                  5         M     UR                  U
5         [        XUU5        UR                  5         UR                  5         S S S 5        UR                   S   S	   UR                   S   S	   :X  d.   S
UR                   S   S	    SUR                   S   S	    S35       eUS-   S-  S:H  =(       d    US-   [        U	5      :H  nUR                  S:  a  [#        UU
U=(       d    UUSS9  U(       a  UR%                  5         UR%                  5         [&        R(                  " SU-   5        GM-     [*        R,                  " 5         g ! , (       d  f       GN= f)NrS   rr   ru   rx   TFrj   r   r>   z:Learning rates found in each optimizer did not align
opt: z

DDP opt: 
r<   )rtolrV   )r	   r   r   rP   r{   rY   r[   r(   r)   r%   r3   stepr_   rZ   num_processesr|   param_groupsr!   	zero_gradr   r]   r
   r}   )rv   rw   rt   rz   ry   r/   r,   rM   rI   rL   rJ   rN   rO   r   r~   ra   rb   r-   r.   _r   s                        r    1test_gradient_accumulation_with_opt_and_schedulerr      s    $>[j#k /mo+%AK
 DVVacgCh@E9y%j1	 %	:#**Iz+BCv!3!34fii@R@R6Sv5%e<
]a1$9q=S_*L

{889AJJL : ##I.yZELLNNN / "4(G,@,@,CD,II 	
I#JZJZ[\J]^bJcIddopw  qE  qE  FG  qH  IM  qN  pO  OQ  R	
I  !mq(Q.WY]s:4V$$q("+O MMO 	$*+W 2X  3 /.s   .I??
J	c                  B   [        5       n [        SS9n[        USS9n[        SS9n[        USS9nU R                  X$5      u  p$U R                  R
                  b   e[        U5       GH  u  pV[        U R                  R
                  5      [        U5      :X  d   eU[        U5      S-
  :  a  U R                  R                  (       a   eUS:X  a  [        U5       H  u  pv[        U R                  R
                  5      [        U5      :X  d   eU[        U5      S-
  :  a  U R                  R                  (       a   eMd  U R                  R                  (       a  M   e   M  M  U R                  R                  (       a  GM   e   U R                  R
                  b   eg )Nr6   r7   r9   r:   `   rj   )
r   r   r   rH   gradient_stateactive_dataloaderr{   idr_   end_of_dataloader)r/   
first_dsetfirst_dataloadersecond_dsetsecond_dataloaderr   r   	batch_nums           r    test_dataloader_breakr   2  s~   -K""-J!*<#2.K";2>*5*=*=>N*b'%%77???!"23	+,,>>?2FVCWWWWs+,q00"11CCCCA~$-.?$@LIk88JJKrRcOdddd 3'8#9A#==#.#=#=#O#OOO#O*99KKKKK %A  --????? 4 %%77???r"   c                  >   [        5       n U R                  nUR                  S:X  a  [        S5        UR                  [
        R                  :w  a
  [        5         UR                  [
        R                  :X  a&  UR                  S:X  a  [        S5        [        U 5        UR                  [
        R                  [
        R                  [
        R                  [
        R                  [
        R                  [
        R                  [
        R                   4;   aL  UR                  S:X  a  [        S5        [#        U 5        UR                  S:X  a  [        S5        [%        U 5        UR                  [
        R                  [
        R                  [
        R                  [
        R                  [
        R                  [
        R                   4;   aM  S HG  nS H>  nS H5  nUR                  S:X  a  [        SSU S	U S
U S35        ['        X#U5        M7     M@     MI     UR                  S:X  a  [        SS5        [)        5         UR                  [
        R                  [
        R                  [
        R                  [
        R                  [
        R                  [
        R                   4;   ad  S H^  nS HU  nS HL  nU(       d  U(       d	  U(       d  M  UR                  S:X  a  [        SSU S	U S
U S35        [)        X#U5        MN     MW     M`     UR+                  5         g )Nr   zA**Test `accumulate` gradient accumulation with dataloader break**z'**Test NOOP `no_sync` context manager**z.**Test Distributed `no_sync` context manager**zE**Test Distributed `no_sync` context manager with multiple forwards**)TFz+**Test `accumulate` gradient accumulation, z`split_batches=z` and `dispatch_batches=z` and `sync_each_batch=z`**zH**Test `accumulate` gradient accumulation with optimizer and scheduler, zJ`split_batches=False`, `dispatch_batches=False`, `sync_each_batch=False`**)r   statelocal_process_indexprintdistributed_typer   XLAr   NOrd   	MULTI_GPU	MULTI_NPU	MULTI_MLU
MULTI_SDAA
MULTI_MUSA	MULTI_CPU	MULTI_HPUrh   rp   r   r   destroy_process_group)r/   r   split_batchrw   rt   s        r    mainr   K  s   -KE  A%QR!4!44!3!33$$);<{#!!!!!!""""!!!!"  $$)BCk*$$)YZ*;7!!!!!!""""!!"  )K$1 '4O00A5I-k]:RScRdd{  }L  |M  MP  Q /{o^ (5 %2 )   A%VX	
 67!!!!!!""""!!"  )K$1 '4O&/? 00A5f-k]:RScRdd{  }L  |M  MP  Q Fketu (5 %2 ) 
!r"   c                     [        5         g r$   )r   )indexs    r    _mp_fnr     s    Fr"   __main__)T)F)FFF)%copyr   r   torch.nn.functionalnn
functionalr&   torch.optimr   torch.optim.lr_schedulerr   torch.utils.datar   accelerate.acceleratorr   r   r	   accelerate.stater
   accelerate.test_utilsr   r   accelerate.utilsr   r   r!   r3   rP   rd   rh   rp   r   r   r   r   r   __name__rA   r"   r    <module>r      s         - ' c c * D 6 #(.>D%>P3l&!T BG7!t@2F"R
 zF r"   