
    /h^                        S SK r S SKrS SKrS SKrS SKJr  S SKJr  S SKr	S SK
r
S SKJrJr  S SKJr  S SKJrJr  S SKJr  S SKJrJr  S S	KJrJrJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%  \#" 5       (       a  S S
KJ&r'  OS SKJ'r'  \ " 5       (       a  Sr(Sr)OSr(Sr)S'S jr*S r+S r,S r-S r.S r/S r0S r1S r2S r3S r4S r5S r6S r7S'S jr8S'S jr9S r:S r;S  r<S! r=S" r>S# r?S$ r@S% rA\BS&:X  a  \A" 5         gg)(    N)deepcopy)Path)
DataLoaderDataset)Accelerator)SeedableRandomSamplerprepare_data_loader)AcceleratorState)RegressionDatasetare_the_same_tensors)DataLoaderConfigurationDistributedTypegathergather_objectis_bf16_availableis_datasets_availableis_fp16_availableis_hpu_availableis_ipex_availableis_pytest_availableis_xpu_availableset_seedsynchronize_rng_states)RegressionModel4XPU)RegressionModelgMbP?gư>c                 b    U(       a  [        UU [        U 5      S9n[        XUS9$ [        XSUS9$ )zBCreates a dataloader that can also use the `SeedableRandomSampler`)	generatordata_sourcenum_samples)
batch_sizesamplerT)r    shuffler   )r   lenr   )	train_setr   r    use_seedable_samplerr!   s        c/home/james-whalen/.local/lib/python3.13/site-packages/accelerate/test_utils/scripts/test_script.pygenerate_baseline_dataloaderr'   ?   s=     (!I

 )GLL)DT]^^    c                 4    [        SU R                   35        g )NzPrinting from the main process printprocess_indexstates    r&   
print_mainr/   N       	+E,?,?+@
ABr(   c                 4    [        SU R                   35        g )Nz%Printing from the local main process )r+   local_process_indexr-   s    r&   print_local_mainr3   R   s    	1%2K2K1L
MNr(   c                 4    [        SU R                   35        g )NPrinting from the last process r*   r-   s    r&   
print_lastr6   V   r0   r(   c                 :    [        SU SU R                   35        g )NPrinting from process : r*   )r.   process_idxs     r&   print_onr;   Z   s    	";-r%2E2E1F
GHr(   c                     [        5       n U R                  n[        S5      nU R                  5          U R                  (       a=  [
        R                  " S5        [        US5       nUR                  S5        S S S 5        O&[        US5       nUR                  S5        S S S 5        S S S 5        U R                  5         U R                  (       a  [        U5       nSR                  UR                  5       5      nS S S 5         WR                  S5      (       d   S5       eUS:  a  UR                  S5      (       d   S5       eUR                  S5      U R                  S-
  :X  d)   S	UR                  S
5      S-    SU R                   35       e U R                  (       a%  UR#                  5       (       a  UR!                  5         U R                  5         [$        R&                  " 5       n[(        R*                  " U5         U R-                  [.        5      " U R0                  5        S S S 5        UR3                  5       R5                  5       nU R                  (       a  US:X  d
   U S35       eO,UR3                  5       R5                  5       S:X  d
   U S35       eUR7                  S5        UR9                  S5        [(        R*                  " U5         U R;                  [<        5      " U R0                  5        S S S 5        U R>                  (       a%  UR3                  5       R5                  5       S:X  d   eO$UR3                  5       R5                  5       S:X  d   eUR7                  S5        UR9                  S5        [(        R*                  " U5         U RA                  [B        5      " U R0                  5        S S S 5        U RD                  (       a?  UR3                  5       R5                  5       SU R0                  R                  S-
   3:X  d   eO$UR3                  5       R5                  5       S:X  d   eUR7                  S5        UR9                  S5        [G        U5       H  n[(        R*                  " U5         U RI                  [J        US9" U R0                  U5        S S S 5        U RL                  U:X  a5  UR3                  5       R5                  5       SU SU RL                   3:X  d   eO$UR3                  5       R5                  5       S:X  d   eUR7                  S5        UR9                  S5        M     g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN{= f! [         a    UR!                  5         e f = f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN<= f)Nzcheck_main_process_first.txt皙?za+zCurrently in the main process
zNow on another process
 zMain process was not first   zOnly wrote to file zNow on another processz times, not z Printing from the main process 0z$ != Printing from the main process 0z != ""r   z&Printing from the local main process 0r5   )r,   r8   r9   )'r   num_processesr   main_process_firstis_main_processtimesleepopenwritewait_for_everyonejoin	readlines
startswithendswithcountAssertionErrorunlinkexistsioStringIO
contextlibredirect_stdouton_main_processr/   r.   getvaluerstriptruncateseekon_local_main_processr3   is_local_main_processon_last_processr6   is_last_processrange
on_processr;   r,   )acceleratorr@   pathftextresultr:   s          r&   process_execution_checkrd   ^   sr   -K--M./D		'	'	)&&JJsOdD!Q9: "! dD!Q23 " 
* !!#""$Z1771;;=)D 		??#DEEcGccEq }}%?@@^B^^@::89[=V=VYZ=ZZ %djj1I&JQ&N%O|\g\u\u[vwZ ""t{{}}!!#
A		#	#A	&##J/0A0AB 
'ZZ\  "F"";;lxGk=ll;zz|""$*=vhf,==*JJqMFF1I		#	#A	&))*:;K<M<MN 
'((zz|""$(PPPPzz|""$***JJqMFF1I		#	#A	&##J/0A0AB 
'""zz|""$*I+J[J[JiJilmJmIn(oooozz|""$***JJqMFF1I]+''*""8;"GHYHY[fg +$$3::<&&(.D[MQST_TmTmSn,oooo::<&&(B...	

1	q	 ,m "! "! 
*	) Z  	KKM	 
'	& 
'	& 
'	& +*s   4U&T98UUU U/BV &V &V2&W;&W9
U	U
U	U
U,/
U>V 
V/2
W
W
W%	c                  d    [        5       n U R                  S:X  a  [        S5        [        U 5        g )Nr   zTesting, testing. 1, 2, 3.)r
   r2   r+   r-   s    r&   init_state_checkrf      s(    E  A%*+	%Lr(   c                     [        5       n [        S/5        [        [        R                  " 5       5      (       d   S5       eU R
                  [        R                  :X  a@  [        S/5        [        [        R                  R	                  5       5      (       d   S5       eO]U R
                  [        R                  :X  a?  [        S/5        [        [        R                  R	                  5       5      (       d   S5       e[        R                  " 5       n[        S/US9  [        UR                  5       5      (       d   S	5       eU R                  S
:X  a  [        S5        g g )Ntorchz*RNG states improperly synchronized on CPU.cudaz*RNG states improperly synchronized on GPU.xpuz*RNG states improperly synchronized on XPU.r   )r   z0RNG states improperly synchronized in generator.r   zAll rng are properly synched.)r
   r   r   rh   get_rng_statedistributed_typer   	MULTI_GPUri   	MULTI_XPUrj   	Generator	get_stater2   r+   )r.   r   s     r&   rng_sync_checkrq      s   EG9% 3 3 566d8dd6!:!::x(#EJJ$<$<$>??mAmm?			?#<#<	<w'#EII$;$;$=>>l@ll>!IK=I>	 3 3 566j8jj6  A%-. &r(   c            	         [        5       n SU R                  -  n[        [        U5      SS9n[	        X R
                  U R                  U R                  SS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      n[        R                  " UR                  5       [        R                  " SU5      R                  5       5      (       d   S5       e[        [        U5      SS9n[	        UU R
                  U R                  U R                  SSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      n[        R                  " UR                  5       [        R                  " SU5      R                  5       5      (       d   S5       eU R                  S:X  a  [        S	5        [        [        U5      SSS
9n[	        X R
                  U R                  U R                  SS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      R!                  5       nUR#                  5         U[%        [        U5      5      :X  d   S5       e[        [        U5      SSS
9n[	        UU R
                  U R                  U R                  SSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      R!                  5       nUR#                  5         U[%        [        U5      5      :X  d   S5       eU R&                  S:X  a  [        S5        g g )N       r    T)put_on_devicer   %Wrong non-shuffled dataloader result.)rv   split_batchesz Non-shuffled dataloader passing.r    r"   !Wrong shuffled dataloader result.zShuffled dataloader passing.r
   r@   r   r]   r	   devicer,   appendr   rh   catequalcpuarangelongr+   tolistsortlistr2   r.   lengthdlrc   batchs        r&   dl_preparation_checkr      s   E%%%%F	E&Ma	0B	Ru/B/BEDWDWgk	lBFfUm$ YYvF;;vzz|U\\!V%<%A%A%CDDmFmmD	E&Ma	0B	

B FfUm$ YYvF;;vzz|U\\!V%<%A%A%CDDmFmmDa01	E&Ma	>B	Ru/B/BEDWDWgk	lBFfUm$ YYv%%'F
KKMT%-((M*MM(	E&Ma	>B	

B FfUm$ YYv%%'F
KKMT%-((M*MM(  A%,- &r(   c            
         [        5       n SU R                  -  n[        [        U5      SS9n[	        X R
                  U R                  U R                  SSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      n[        R                  " UR                  5       [        R                  " SU5      R                  5       5      (       d   S5       e[        [        U5      SS9n[	        UU R
                  U R                  U R                  SSSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      n[        R                  " UR                  5       [        R                  " SU5      R                  5       5      (       d   S5       eU R                  S:X  a  [        S	5        [        [        U5      SSS
9n[	        X R
                  U R                  U R                  SSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      R!                  5       nUR#                  5         U[%        [        U5      5      :X  d   S5       e[        [        U5      SSS
9n[	        UU R
                  U R                  U R                  SSSS9n/ nU H  nUR                  [        U5      5        M     [        R                  " U5      R!                  5       nUR#                  5         U[%        [        U5      5      :X  d   S5       eU R&                  S:X  a  [        S5        g g )Nrs   rt   ru   T)rv   dispatch_batchesr   rw   )rv   rx   r   z(Non-shuffled central dataloader passing.ry   rz   z$Shuffled central dataloader passing.r{   r   s        r&   central_dl_preparation_checkr      s   E%%%%F	E&Ma	0B	
LL%--u/B/BRVim
B FfUm$ YYvF;;vzz|U\\!V%<%A%A%CDDmFmmD	E&Ma	0B	

B FfUm$ YYvF;;vzz|U\\!V%<%A%A%CDDmFmmDa89	E&Ma	>B	
LL%--u/B/BRVim
B FfUm$ YYv%%'F
KKMT%-((M*MM(	E&Ma	>B	

B FfUm$ YYv%%'F
KKMT%-((M*MM(  A%45 &r(   c                     [        5       n  " S S[        5      n " S S5      nU" [        SU R                  -  5      5      nU" [	        U5      SS9n[        X4S9n[        XPR                  U R                  U R                  5      n[        UR                  S	5      (       a-  [        UR                  R                  U5      (       d   S
5       eg [        UR                  U5      (       d   S
5       eg )Nc                   &    \ rS rSrS rS rS rSrg)+custom_sampler_check.<locals>.CustomDatasetiA  c                     Xl         g Ndata)selfr   s     r&   __init__4custom_sampler_check.<locals>.CustomDataset.__init__B  s    Ir(   c                 ,    [        U R                  5      $ r   )r#   r   r   s    r&   __len__3custom_sampler_check.<locals>.CustomDataset.__len__E  s    tyy>!r(   c                      U R                   U   $ r   r   )r   indexs     r&   __getitem__7custom_sampler_check.<locals>.CustomDataset.__getitem__H  s    99U##r(   r   N)__name__
__module____qualname____firstlineno__r   r   r   __static_attributes__ r(   r&   CustomDatasetr   A  s    		"	$r(   r   c                   :    \ rS rSrS
S\S\S\4S jjrS rS rSr	g	)0custom_sampler_check.<locals>.CustomBatchSampleriK  dataset_lengthr    r"   c                 R    X l         [        R                  " U5      U l        X0l        g r   )r    npr   
data_indexr"   )r   r   r    r"   s       r&   r   9custom_sampler_check.<locals>.CustomBatchSampler.__init__L  s    (O ii7DO"Lr(   c              3      #    [        U 5      nU R                  (       a*  [        R                  R	                  U R
                  5      nOU R
                  n[        R                  " X!5      nU S h  vN   g  N7fr   )r#   r"   r   randompermutationr   array_split)r   num_batchesr   outputs       r&   __iter__9custom_sampler_check.<locals>.CustomBatchSampler.__iter__Q  sM     d)K||		--doo>^^E7Fs   A.A80A61A8c                 n    [         R                  " [        U R                  5      U R                  -  5      $ r   )mathceilr#   r   r    r   s    r&   r   8custom_sampler_check.<locals>.CustomBatchSampler.__len__Z  s#    99S1DOOCDDr(   )r    r   r"   N)T)
r   r   r   r   intboolr   r   r   r   r   r(   r&   CustomBatchSamplerr   K  s'    	#3 	#C 	#$ 	#
		Er(   r   rs   rt   ru   )batch_samplerr   z>Custom sampler was changed after calling `prepare_data_loader`)r
   r   r]   r@   r#   r   r	   r|   r,   hasattrr   
isinstance)r.   r   r   datasetr!   r   s         r&   custom_sampler_checkr   >  s    E$ $E E$ E"u':':":;<G W!<G	G	3B	Ru/B/BEDWDW	XBr11"**88:LMM 	
L	
M "**,>?? 	
L	
?r(   c                  "   [        S5        [        SSS9n [        U SSS9n[        SS9n[	        US9nUR                  U5      n/ n[        S	5       H   nU H  nUR                  US
   5        M     M"     [        R                  " U5      n[        S5        UR                  S5        / n[        S	5       H   nU H  nUR                  US
   5        M     M"     [        R                  " U5      n[        R                  " XG5      (       d   S5       eg )N*   
   r   seed   Try   r%   dataloader_config   xr   z;Did not obtain the same items with the same seed and epoch.)r   r   r   r   r   preparer]   r}   rh   r~   	set_epochallclose)r$   train_dlconfigr_   original_items_r   	new_itemss           r&   check_seedable_samplerr   l  s    RL!"5I)4@H$$?F7K""8,HN1XE!!%*-   YY~.N RLqI1XEU3Z(   		)$I>>.44s6ss4r(   c                  2   [        S5        [        SS9n [        U S9nUR                  S:  d   S5       e[	        [        [        S5      5      SSS9n[        USS	9nUR                  R                  R                  n[        U[        5      (       d   S
5       eg )Nr   Tr   r   r?   z)This test requires more than one process.r   ry   )
dataloaderr%   z:Sampler in BatchSamplerShard is not SeedableRandomSampler.)r   r   r   r@   r   r   r]   r	   r   r!   r   r   )r   r_   r   prepared_data_loadertarget_samplers        r&   -check_seedable_sampler_in_batch_sampler_shardr     s    RL$$?F7K$$q(U*UU(DrO4HJ.!
 *77EEMMNn&;<< D<r(   c                  2   [        S5        Sn [        SSS9n[        USSS9n[        SU S9n[	        US9nUR                  U5      n/ n[        S	5       H   nU H  nUR                  US
   5        M     M"     [        R                  " U5      nSUl
        [	        US9nUR                  U5      n/ n	[        S	5       H   nU H  nU	R                  US
   5        M     M"     [        R                  " U	5      n	[        R                  " Xi5      (       a   S5       eg )Nr   r   r   r   Try   )r%   	data_seedr   r   r   +   z1Obtained the same items with different data seed.)r   r   r   r   r   r   r]   r}   rh   r~   r   r   )
r   r$   r   r   r_   prepared_dlr   r   r   r   s
             r&   %check_seedable_sampler_with_data_seedr     s   RLI!"5I)4@H$$)TF7K%%h/KN1X E!!%*- !  YY~.N F7K%%h/KI1X EU3Z( !  		)$I~~n88m:mm88r(   c                    [        S5        UR                  S5        [        U SS9n[        XBX5      n[	        5       n[
        R                  R                  UR                  5       SS9n[        S5       Hs  nU Hj  n	UR                  5         U" U	S   5      n
[
        R                  R                  R                  XS   5      nUR                  5         UR                  5         Ml     Mu     XF4$ )Nr   r   r=   lrr   r   y)r   manual_seedr   r'   r   rh   optimSGD
parametersr]   	zero_gradnn
functionalmse_lossbackwardstep)r   r    r   r%   r$   r   model	optimizerepochr   r   losss               r&   mock_trainingr     s    RL"!b9I+I*cHE 0 0 2s;IqEOO5:&F88&&//c
CDMMONN   r(   c                    [        5       n[        R                  " 5       nSnUS-  UR                  -  n[	        XCUR                  -  X 5      u  pV[        UR                  5      (       d   S5       e[        UR                  5      (       d   S5       e[        5       n[        XRX05      n[        5       n	[        R                  R                  U	R                  5       SS9n
UR                  XU
5      u  pn
[        S5        UR!                  S5        [#        S5       Ht  nU Hk  nU	R%                  5         U	" US   5      n[        R&                  R(                  R+                  XS	   5      nUR-                  U5        U
R/                  5         Mm     Mv     UR1                  U	5      R3                  5       n	[        R4                  R7                  UR                  U	R                  [8        [:        S
 S9  [        R4                  R7                  UR                  U	R                  [8        [:        S S9  UR=                  S5        [?        SU S9n[        US9n[        XRX1R                  -  U 5      n[        5       n	[        R                  R                  U	R                  5       SS9n
UR                  XU
5      u  pn
[        S5        UR!                  S5        [#        S5       Ht  nU Hk  nU	R%                  5         U	" US   5      n[        R&                  R(                  R+                  XS	   5      nUR-                  U5        U
R/                  5         Mm     Mv     UR1                  U	5      R3                  5       n	[        R4                  R7                  UR                  U	R                  [8        [:        S S9  [        R4                  R7                  UR                  U	R                  [8        [:        S S9  UR=                  S5        [        R@                  RC                  5       (       a  [=        S5        [         RD                  " 5         [        SS9n[        R&                  RG                  SS5      n	UR                  U	5      n	UR1                  U	SS9n[        RH                  " SS/5      RK                  [        RL                  URN                  S9nU" U5      n[Q        5       (       Ga  [=        S5        [         RD                  " 5         [?        U S9n[        SUS9n[        XRX05      n[        5       n	[        R                  R                  U	R                  5       SS9n
UR                  XU
5      u  pn
[        S5        UR!                  S5        [#        S5       Ht  nU Hk  nU	R%                  5         U	" US   5      n[        R&                  R(                  R+                  XS	   5      nUR-                  U5        U
R/                  5         Mm     Mv     UR1                  U	5      R3                  5       n	[        R4                  R7                  UR                  U	R                  [8        [:        S S9  [        R4                  R7                  UR                  U	R                  [8        [:        S  S9  [S        5       (       Ga  [U        5       (       Gd  [=        S!5        [         RD                  " 5         [?        U S9n[        SUS9n[        XRX05      n[        5       n	[        R                  R                  U	R                  5       SS9n
UR                  XU
5      u  pn
[        S5        UR!                  S5        [#        S5       Ht  nU Hk  nU	R%                  5         U	" US   5      n[        R&                  R(                  R+                  XS	   5      nUR-                  U5        U
R/                  5         Mm     Mv     UR1                  U	5      R3                  5       n	[        R4                  R7                  UR                  U	R                  [8        [:        S" S9  [        R4                  R7                  UR                  U	R                  [8        [:        S# S9  [W        5       (       Ga  [=        S$5        [         RD                  " 5         [?        U S9n[        SSUS%9n[        XRX05      n[        5       n	[        R                  R                  U	R                  5       SS9n
UR                  XU
5      u  pn
[        S5        UR!                  S5        [#        S5       Ht  nU Hk  nU	R%                  5         U	" US   5      n[        R&                  R(                  R+                  XS	   5      nUR-                  U5        U
R/                  5         Mm     Mv     UR1                  U	5      R3                  5       n	[        R4                  R7                  UR                  U	R                  [8        [:        S& S9  [        R4                  R7                  UR                  U	R                  [8        [:        S' S9  g g )(Nrt      z0Did not obtain the same model on both processes.r=   r   r   r   r   r   c                     SU  3$ Nz>Did not obtain the same model on CPU or distributed training.
r   msgs    r&   <lambda> training_check.<locals>.<lambda>      YZ]Y^_r(   )atolrtolr   c                     SU  3$ r   r   r   s    r&   r   r     r   r(   zVTraining yielded the same results on one CPU or distributed setup with no batch split.T)rx   r%   r   c                     SU  3$ r   r   r   s    r&   r   r     r   r(   c                     SU  3$ r   r   r   s    r&   r   r     r   r(   zSTraining yielded the same results on one CPU or distributed setup with batch split.zKeep fp32 wrapper check.fp16)mixed_precisionr   )keep_fp32_wrapperr?   )dtyper|   zBF16 training check.r   bf16)r  r   c                     SU  3$ r   r   r   s    r&   r   r   D      ]^a]bcr(   c                     SU  3$ r   r   r   s    r&   r   r   K  r  r(   zFP16 training check.c                     SU  3$ r   r   r   s    r&   r   r   j  r  r(   c                     SU  3$ r   r   r   s    r&   r   r   q  r  r(   zipex BF16 training check.)r  r   r   c                     SU  3$ r   r   r   s    r&   r   r     r  r(   c                     SU  3$ r   r   r   s    r&   r   r     r  r(   ),r
   rh   ro   r@   r   r   abr   r'   r   r   r   r   r   r   r   r]   r   r   r   r   r   r   unwrap_modelr   testingassert_closeATOLRTOLr+   r   ri   is_available_reset_stateLinearTensortofloat16r|   r   r   r   r   )r%   r.   r   r    r   r$   	old_modelr_   r   r   r   r   r   r   r   r   model_with_fp32_wrapperinput_tensors                     r&   training_checkr    sk   E!IJ!^e111F(e>Q>Q1QS\sI	,,`.``,	,,`.``,-K+I*cHE 0 0 2s;I!,!4!4Xi!PHYRL"1XEOO5:&F88&&//c
CD  &NN   $$U+//1E	MM_   
MM_   no/dYmn0ABK+j+>+>>@TH E 0 0 2s;I!,!4!4Xi!PHYRL"1XEOO5:&F88&&//c
CD  &NN   $$U+//1E	MM_   
MM_   kl zz  ()%%'!&91%##E*"-":":5TX":"Y ||QF+..U]];K]K].^(6 $%%%'3I]^!&L]^/	jg!KKOOE$4$4$63O?	%0%8%8)%T"b!qA!!uSz*xx**33F#JG$$T*  "  ((/335""KKGGc 	# 	
 	""KKGGc 	# 	
 #3#5#5$%%%'3I]^!&L]^/	jg!KKOOE$4$4$63O?	%0%8%8)%T"b!qA!!uSz*xx**33F#JG$$T*  "  ((/335""KKGGc 	# 	
 	""KKGGc 	# 	
 )*%%'3I]^!&dVgh/	jg!KKOOE$4$4$63O?	%0%8%8)%T"b!qA!!uSz*xx**33F#JG$$T*  "  ((/335""KKGGc 	# 	
 	""KKGGc 	# 	
9 r(   c           	         [        5       nU R                  [        SUR                  -  5       Vs/ s H  n[	        US9PM     sn5      nUR                  USS9 n[        U5      S:X  d    SUR                   S[        U5       35       e S S S 5        U R                  [        SUR                  -  S-
  5       Vs/ s H  n[	        US9PM     sn5      nUR                  USS9 nUR                  (       a0  [        U5      S:X  d    SUR                   S[        U5       35       eO/[        U5      S:X  d    S	UR                   S[        U5       35       eS S S 5        UR                  5         U R                  [        SUR                  -  S-
  5       Vs/ s H  n[	        US9PM     sn5      nU R                  [        SUR                  -  5       Vs/ s H  n[	        US9PM     sn5      nXV4 GH  nUS
   nUR                  USS9 nUR                  S:X  a9  [        U5      [        U5      :X  d    SUR                   S[        U5       35       eO/[        U5      S:X  d    SUR                   S[        U5       35       e/ nU H  n	UR                  U	5        M     S S S 5        UR                  5         [        W5      n
U
S [        U5        Vs/ s H  oS
   PM	     nnX|:X  a  M   SU SU 35       e   g s  snf ! , (       d  f       GN{= fs  snf ! , (       d  f       GN= fs  snf s  snf ! , (       d  f       N= fs  snf )Nr   )kFapply_padding4Each process did not have two items. Process index: 
; Length: r?   z;Last process did not receive a single item. Process index: zLOne of the intermediate processes did not receive two items. Process index: r  Tz9Single process did not receive all items. Process index: )Gathered results is incorrect. Expected: ; Got: )r
   	from_listr]   r@   dictsplit_between_processesr#   r,   r\   rG   r}   r   )datasets_Datasetr.   vr   resultsodd_data	even_dataexpected_outputresults_per_processrc   gathered_resultsrr   s                r&   $test_split_between_processes_datasetr2    sc   E%%%EDWDW@W:X&Y:XQtay:X&YZD		&	&t5	&	AW7|q  	
B5CVCVBWWabefmbnaop	
  
B
 %%%EDWDW@WZ[@[:\&]:\Qtay:\&]^D		&	&t5	&	AW  w<1$ MeNaNaMbblmpqxmylz{$ w<1$ ^_d_r_r^ss}  B  CJ  K  ~L  M$ 
B 
))eAH[H[D[^_D_>`*a>`4!9>`*abH **uQI\I\E\?]+^?]!D1I?]+^_I%s)**4t*D""a'7|s4y0 OPUPcPcOddnorszo{n|}0 7|q( J5K^K^J__ijmnujviwx( #%!#**62 " E 	!()<="2;SY"?@"?QC&"?@(v,UVeUffmntmu*vv(- &+ 'Z	A	A
 '^	A	A +b+^
 ED" AsC   K:0K?;L%A1LL(L-BL2M?
L
L%2
M 	c                     [        5       n [        [        SSU R                  -  5      5      nU R	                  U5       n[        U5      S:X  d    SU R                   S[        U5       35       e S S S 5        U R                  5         [        [        SSU R                  -  5      5      n[        [        SSU R                  -  S-
  5      5      nXC4 H  nUnU R	                  USS9 n[        R                  " [        U5      U R                  -  5      n[        U5      U:X  d-   SU R                   S	U R                   S[        U5       35       e/ nU H  nUR                  U5        M     S S S 5        U R                  5         [        W5      n	U	S [        U5       n
XZ:X  a  M   S
U SU
 35       e   g ! , (       d  f       GNK= f! , (       d  f       N`= f)Nr   r   r"  r#  r?   Tr   Process ; did not get the correct number of item(s). Process index: r$  r%  )r
   r   r]   r@   r(  r#   r,   rG   r   r   r|   r}   r   )r.   r   r+  r-  r,  r.  num_samples_per_devicer/  rc   r0  r   s              r&   !test_split_between_processes_listr7    s   EaU00012D		&	&t	,7|q  	
B5CVCVBWWabefmbnaop	
  
- 
U1q5#6#6689IE!a%"5"55:;<H%**4t*D%)YYs4y5;N;N/N%O"w<#99 5<<.(cdidwdwcx  yC  DG  HO  DP  CQ  R9 #%!#**62 " E 	!()<=!+CI.(v,UVeUffmntmu*vv(' & 
-	, EDs   0F3&BG3
G
G	c                     [        5       n / SQn/ SQn[        R                  " / SQ5      nU R                  S;   Ga  XUS.n[	        U5      nU R                  U5       nU R                  S:X  a  US   US   S SU R                  -   :X  d   eOWU R                  S:X  a  US   US   S	S  :X  d   eO5U R                  S
:X  a%  US   US   SS  :X  d   SUS   S    SUS    35       eU R                  S:X  a  US   US   S SU R                  -   :X  d   eOCU R                  S:X  a  US   US   S	S  :X  d   eO!U R                  S
:X  a  US   US   SS  :X  d   eU R                  S:X  aV  [        R                  " US   US   S SU R                  -   5      (       d#   SUS   S SU R                  -    SUS    35       eOU R                  S:X  a<  [        R                  " US   US   S	S  5      (       d   SUS   S	S   SUS    35       eOKU R                  S
:X  a;  [        R                  " US   US   SS  5      (       d   SUS   SS   SUS    35       eS S S 5        U R                  5         g ! , (       d  f       N= f)N)r?   r   r   r            rt   )r  r  cdera   gh)r?   r   r   )r  r  r<  r   r  rt   r   r   r   z
Expected: 
, Actual: r  r<  z7Did not obtain expected values on process 0, expected `z`, received: z7Did not obtain expected values on process 2, expected `z7Did not obtain expected values on process 4, expected `)	r
   rh   tensorr@   r   r(  r,   r   rG   )r.   r  r  r<  r   	data_copyr+  s          r&   (test_split_between_processes_nested_dictrE    s
   E A0A-.Ai'Q'TN	**40G""a's|y~6PU=P=P8P'QQQQ$$)s|y~ab'9999$$)s|y~bc'::ujSVXZI[H\\fgnorgsft<uu:""a's|y~6PU=P=P8P'QQQQ$$)s|y~ab'9999$$)s|y~bc'::::""a'~~gclIcNC]Q%J]J]E]4^__ MdSViXrZ[_d_r_rZrNsMt  uB  CJ  KN  CO  BP  Q_ $$)~~gclIcN124FGG MdSViXYXZm_\ijqrujviwxG $$)~~gclIcN234GHH MdSViXZX[nM]]jkrsvkwjxyH/ 16 
7 10s   G6I,,
I:c                  .   [        5       n U R                  S:  a  [        R                  " / SQ/ SQ/5      R	                  U R
                  5      nU R                  U5       nU R                  S:X  a3  [        R                  " / SQ/5      R	                  U R
                  5      nO2[        R                  " / SQ/5      R	                  U R
                  5      n[        R                  R                  X#5        S S S 5        U R                  5         [        R                  " [        SU R                  -  5       Vs/ s H  oD/PM     sn5      R	                  U R
                  5      n[        R                  " [        SU R                  -  S-
  5       Vs/ s H  oD/PM     sn5      R	                  U R
                  5      nXV4 GH  nUR                  5        Vs/ s H  n[        R                  " U5      PM     nnU R                  USS9 n[        R                  " [        U5      U R                  -  5      n[        U5      U:X  d-   SU R
                   S	U R                   S
[        U5       35       e/ n	U H#  n
U	R!                  U
R	                  S5      5        M%     S S S 5        U R                  5         [#        W	5      nUS [        U5       nX|:X  a  GM   SU SU 35       e   g ! , (       d  f       GN= fs  snf s  snf s  snf ! , (       d  f       Np= f)Nr?   )r   r?   r   r   )r   r9  r:  r;  r   r   Tr   r4  r5  r#  r   r$  r%  )r
   r@   rh   rC  r  r|   r(  r,   r  r  rG   r]   r   r   r   r#   r}   r   )r.   r   r+  expectedir-  r,  r.  r6  r/  rc   r0  r   s                r&   #test_split_between_processes_tensorrI    s   EQ||\<89<<U\\J**40G""a' <<7::5<<H <<7::5<<HMM&&w9 1 	!5U5H5H1H+IJ+Iac+IJKNNu||\I||%E4G4G0G!0K*LM*LQS*LMNQQRWR^R^_H%48KKMBMq5<<?MB**4t*D%)YYs4y5;N;N/N%O"w<#99 5<<.(cdidwdwcx  yC  DG  HO  DP  CQ  R9 #%!#**699U+;< " E 	!()<=!+CI.(v,UVeUffmntmu*vv(# & 10 KMBDDs,   !BK%8K7K< LBL%
K4
L	c                     [        5       n U R                  S;   a  [        [        S5      5      n[	        U5      U R                  -  n[	        U5      U R                  -  nU R                  U5       nU R                  U:  a,  [	        U5      US-   :X  d   SUS-    S[	        U5       35       eO%[	        U5      U:X  d   SU S[	        U5       35       eS S S 5        U R                  5         g ! , (       d  f       N= f)N)r?   r   r   rt      r?   z2Each Process should have even elements. Expected: rB  )r
   r@   r   r]   r#   r(  r,   rG   )r.   r   num_samples_per_process
num_extrasr+  s        r&   #test_split_between_processes_evenlyrN  3  s    El*E"I"%d)u/B/B"BY!4!44
**40G""Z/7|'>'BB HI`cdIdHeeopst{p|o}~B 7|'>> HI`Haaklopwlxkyz> 1 
 10s   0A"C++
C9c                      [        5       n U R                  5       SL d   eU R                  (       a  U R                  5         U R                  5       SL d   eU R                  5       SL d   eg )NFT)r   check_triggerrB   set_trigger)r_   s    r&   test_triggerrR  E  si    -K$$&%/// ""! $$&$... $$&%///r(   c                     SS K n [        R                  " 5         [        R                  R                  SS5      n[        5       n[        R                  " 5         U R                  [        5       nUR                  U5        S S S 5        S[        WR                  R                  S   5      ;   d   eS[        UR                  R                  S   5      ;   d   eg ! , (       d  f       N_= f)Nr   r?   z*`AcceleratorState` object has no attributez1This happens if `AcceleratorState._reset_state()`)pytestr
   r  rh   r   r  r   raisesAttributeErrorr   strvalueargs)rT  simple_modelr_   cms       r&   test_reinstantiated_stater\  V  s    !!#88??1a(L-K!!#	~	&"L) 
'73rxx}}Q?O;PPPP>#bhhmmTUFVBWWWW 
'	&s   .C
C'c                  
   [        5       n U R                  nUR                  S:X  a  [        S5        [	        5         UR                  5         UR                  [        R                  :X  a  [        R                  R                  5       nOUR                  nX!R                  :X  Ga   UR                  S:X  a  [        S5        [        5         UR                  S:X  a  [        S5        [        5         UR                  S:X  a  [        S5        [!        5         UR                  S:X  a  [        S5        [#        5         UR                  S:X  a  [        S5        [%        5         UR                  S:X  a  [        S5        ['        5       (       a  SS	KJn  [-        U5        O[        S
5        UR                  S:X  a  [        S5        [/        5         UR                  S:X  a  [        S5        [1        5         UR                  [        R2                  :w  a(  [5        5         [7        5         [9        5         [;        5         UR                  S:  a
  [=        5         UR                  [        R>                  :X  a  g UR                  S:X  a  [        S5        [A        SS9  [A        SS9  UR                  S:X  a  [        S5        [C        5         [E        5       (       a%  UR                  S:X  a  [        S5        [G        5         URI                  5         g )Nr   z**Initialization**z
**Test process execution**z+
**Test split between processes as a list**z+
**Test split between processes as a dict**z-
**Test split between processes as a tensor**z(
**Test split between processes evenly**z7
**Test split between processes as a datasets.Dataset**)r   z6Skipped because Hugging Face datasets is not availablez1
**Test random number generator synchronization**z 
**DataLoader integration test**r?   z
**Training integration test**Fr   Tz
**Breakpoint trigger test**z
**Test reinstantiated state**)%r   r.   r2   r+   rf   rG   rl   r   rm   rh   ri   device_countr@   r,   rd   r7  rE  rI  rN  r   datasetsr   r2  rq   r   XLAr   r   r   r   r   	DEEPSPEEDr  rR  r   r\  destroy_process_group)r_   r.   num_processes_per_noder)  s       r&   mainrd  f  sL   -KE  A%"#	!:!::!&!8!8!:!&!4!4 !4!44!#01!!#@A)+!#@A02!#BC+-!#=>+-!#LM ""<01ABJK  A%BC  A%12!4!44$& -/Q57 !:!::  A%/0.-  A%-.N$$)34!#	!r(   __main__)F)CrR   rP   r   rC   copyr   pathlibr   numpyr   rh   torch.utils.datar   r   
accelerater   accelerate.data_loaderr   r	   accelerate.stater
   accelerate.test_utilsr   r   accelerate.utilsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r'   r/   r3   r6   r;   rd   rf   rq   r   r   r   r   r   r   r   r  r2  r7  rE  rI  rN  rR  r\  rd  r   r   r(   r&   <module>ro     s  "  	       0 " M - I   $ L5DDDD_COCIFR/$9.x>6B+
\t4&n8$P
f-w`wB#LwD$0"X N"b zF r(   