
    h                         S SK r S SKrS SKJr  S SKJr  S SKJr  S SKr	S SK
Jr  S SKJr  S SKJr  S SKJr  S S	KJrJr  S S
KJr  S SKJrJr  \" \5      rS r " S S\5      rg)    N)ArgumentParser)	Generator)rmtree)DatasetBuilder)BaseDatasetsCLICommand)DownloadMode)DatasetInfosDict)dataset_module_factoryget_dataset_builder_class)VerificationMode)ERROR
get_loggerc                    [        U R                  U R                  U R                  U R                  U R
                  U R                  =(       d    U R                  U R                  U R                  U R                  U R                  5
      $ )N)TestCommanddatasetname	cache_dirdata_dirall_configs	save_info
save_infosignore_verificationsforce_redownloadclear_cachenum_proc)argss    P/home/james-whalen/.local/lib/python3.13/site-packages/datasets/commands/test.py_test_command_factoryr      sf    		)$//!!     c                   h    \ rS rSrSr\S\4S j5       rS\S\S\S\S	\	S
\	S\	S\	S\	S\
4S jrS rSrg)r   #   Fparserc                    U R                  SSS9nUR                  S[        S SS9  UR                  S[        S SS9  UR                  S	[        S S
S9  UR                  SSSS9  UR                  SSSS9  UR                  SSSS9  UR                  SSSS9  UR                  SSSS9  UR                  S[        S SS9  UR                  SSSS9  UR                  S[        SS9  UR	                  [
        S9  g )NtestzTest dataset loading.)helpz--namezDataset processing name)typedefaultr%   z--cache_dirz.Cache directory where the datasets are stored.z
--data_dirz@Can be used to specify a manual directory to get the files from.z--all_configs
store_truezTest all dataset configurations)actionr%   z--save_infoz6Save the dataset infos in the dataset card (README.md)z--ignore_verificationsz1Run the test without checksums and splits checks.z--force_redownloadzForce dataset redownloadz--clear_cachezBRemove downloaded files and cached datasets after each config testz
--num_proczNumber of processesz--save_infoszalias to save_infor   zName of the dataset to download)r&   r%   )func)
add_parseradd_argumentstrintset_defaultsr   )r"   test_parsers     r   register_subcommandTestCommand.register_subcommand&   sN   ''5L'M  THa b  A	 	! 	
 	  S	 	! 	
 	  Lm n  ,5m 	! 	
 	  $D 	! 	

 	  !5lQk l  U 	! 	

 	  CLa b  K_ `  ;\ ]  &; <r   r   r   r   r   r   r   r   r   r   r   c                     Xl         X l        X0l        X@l        XPl        X`l        Xpl        Xl        Xl        Xl	        U	(       a  U(       d  [        S5        [        S5        U(       a  SU l        g g )Na  When --clear_cache is used, specifying a cache directory is mandatory.
The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.
Please provide a --cache_dir that will be used to test the dataset.   T)_dataset_name
_cache_dir	_data_dir_all_configs_save_infos_ignore_verifications_force_redownload_clear_cache	_num_procprintexit)selfr   r   r   r   r   r   r   r   r   r   s              r   __init__TestCommand.__init__K   sg      
#!'%%9"!1'!yV
 G)-D& r   c           
      .  ^ ^^	^
 [         R                  " S5      R                  [        5        T R                  b'  T R
                  (       a  [        S5        [        S5        T R                  T R                  snm	[        U5      m
[        T
5      mT R
                  (       a&  TR                  (       a  [        TR                  5      OSnS[        [        S S 4   4UU	U
U 4S jjn[        U" 5       5       GH  u  pE[        SUR                   R"                   SUS-    SU S	35        [$        R&                  R)                  [$        R&                  R+                  UR-                  5       [.        R                   R0                  5      5      Ul        UR5                  T R6                  (       d  [8        R:                  O[8        R<                  T R>                  (       a  [@        RB                  O[@        RD                  T RF                  S
9  URI                  5         T RJ                  (       a  [$        R&                  RM                  U5      (       d  [$        R&                  RO                  U5      OUn[$        RP                  " USS9  [S        S0 UR                   R"                  URT                  0D6RW                  U5        [        S[$        R&                  R+                  U[.        R                   RX                  5       35        T RZ                  (       d  GM   [$        R&                  RM                  UR\                  5      (       a7  [^        Ra                  SUR\                   35        [c        UR\                  5        [$        R&                  R+                  T R\                  [.        R                   Rd                  5      n[$        R&                  RM                  U5      (       d  GM  [^        Ra                  SU 35        [c        U5        GM     [        S5        g )NfilelockzABoth parameters `config` and `all_configs` can't be used at once.r4   returnc               3   F  >#    TR                   (       a  TR                  (       a  [        TR                  5       Hx  u  pSTR                  ;   a-  T" STR                  TR
                  S.TR                  D6v   MB  T" SUR                  TR                  TR
                  S.TR                  D6v   Mz     g STR                  ;   a,  T" STR                  TR
                  S.TR                  D6v   g T" STTR                  TR
                  S.TR                  D6v   g 7f)Nconfig_name)r   r   )rH   r   r    )r9   BUILDER_CONFIGS	enumeratebuilder_kwargsr7   r8   r   )iconfigbuilder_clsrH   modulerA   s     r   get_builders%TestCommand.run.<locals>.get_buildersv   s      [%@%@!*;+F+F!GIA$(=(==) &*oo%)^^ %33  * (.&*oo%)^^ %33	  "H !F$9$99%r$..r\b\q\qrr% $/"&//!% !//	 s   DD!zTesting builder 'z' (/))download_modeverification_moder   T)exist_okzDataset card saved at zClearing cache at zTest successful.rI   )3logging	getLoggersetLevelr   r6   r9   r?   r@   r5   r
   r   rJ   lenr   r   rK   rN   r   ospathexistsjoinget_imported_module_dirdatasetsDATASETDICT_INFOS_FILENAME_record_infosdownload_and_preparer<   r   REUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADr;   r   	NO_CHECKS
ALL_CHECKSr>   
as_datasetr:   isdirbasenamemakedirsr	   infowrite_to_directoryREPOCARD_FILENAMEr=   r7   loggerwarningr   DOWNLOADED_DATASETS_DIR)rA   r]   
n_buildersrQ   jbuildersave_infos_dirdownload_dirrO   rH   rP   s   `       @@@r   runTestCommand.runl   s   *%..u5::!d&7&7UVG MM4::k'-/79=9J9J{OjOjS445pq
	id(BC 	 	6 $LN3JA%gnn&9&9%:#a!eWAj\QRST$&GGNNW<<>@j@jk%G! ((-- +@@!22-- #3"<"<%00 )   
 ?Aww}}T?R?R!1!1$!7X\NT: GGNN$7$7#FGZZ[ij.rww||NHOOLmLm/n.opq    77==!3!344NN%78J8J7K#LM7--.!ww||DOOX__=d=de77==..NN%7~#FG<(C 4F 	 !r   )
r9   r7   r=   r8   r5   r<   r;   r6   r>   r:   N)__name__
__module____qualname____firstlineno____test__staticmethodr   r1   r-   boolr.   rB   rx   __static_attributes__rI   r   r   r   r   #   s    H"=N "= "=H.. . 	.
 . . . #. . . .BH"r   r   )rX   r\   argparser   collections.abcr   shutilr   datasets.configra   datasets.builderr   datasets.commandsr   "datasets.download.download_managerr   datasets.infor	   datasets.loadr
   r   datasets.utils.info_utilsr   datasets.utils.loggingr   r   rz   rp   r   r   rI   r   r   <module>r      sL     	 # %   + 4 ; * K 6 4 
H	Q"( Q"r   