hc}^SSKrSSKrSSKJr SSKJr SSKJrJr SSK J r J r J r SSK r SSKJr SSK Jr SSKJr SS KJr S S KJr S S KJr S S KJr S SKJr S SKJr S SKJr S SK!J"r"J#r#J$r$J%r%J&r& S SK'J(r( \ \)\*\*4\)\*\)S4r+\*"\RX5r-\R\"\/5r0"SS\*5r1"SS\25r3Sr4\RXSS/\Rj/SQ\Rl/SQ0r7Sr8\Rr\Rt"S5:a SS /r;/S!Qr\I\*S9\ \S?\ \QS-\I\+4S@jjrR"SAS0\I\*5rS"SBSC\H\*\S45rT"SDSE\I\*5rU"SFSG\H\*\U45rVgs snnfs snnnfs snnfs snnnf)JN)partial) has_magic)PathPurePath)CallableOptionalUnion) url_to_fs) HfFileSystem)version) thread_map)config)DownloadConfig) _split_re)Split)logging)tqdm)!_prepare_path_and_storage_options is_local_pathis_relative_path xbasenamexjoin)string_to_dictc\rSrSrSrg)Url!rN__name__ __module__ __qualname____firstlineno____static_attributes__rM/home/james-whalen/.local/lib/python3.13/site-packages/datasets/data_files.pyrr!r%rc\rSrSrSrg)EmptyDatasetError%rNrrr%r&r)r)%r'r%r)zFdata/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*traintraining) validationvaliddevval)testtestingeval evaluationz-._ 0-9z2023.9.0z**[{sep}/]{keyword}[{sep}]*z{keyword}[{sep}]*)z {keyword}/**z{keyword}[{sep}]*/**z**[{sep}/]{keyword}/**z**[{sep}/]{keyword}[{sep}]*/**z 2023.12.0z**/*[{sep}/]{keyword}[{sep}]*)z{keyword}/**/*z{keyword}[{sep}]*/**/*z**/*[{sep}/]{keyword}/**/*z"**/*[{sep}/]{keyword}[{sep}]*/**/*z**/{keyword}[{sep}]*z**/*[{sep}]{keyword}[{sep}]*)z**/{keyword}/**z**/{keyword}[{sep}]*/**z**/*[{sep}]{keyword}/**z**/*[{sep}]{keyword}[{sep}]*/**)keywordsepz**z*[])z README.mdz config.jsonzdataset_info.jsonzdataset_infos.jsonzdummy_data.zipzdataset_dict.jsonpatternreturnc6^[U4Sj[55$)Nc3,># UH oT;v M g7fNr).0wildcard_characterr7s r& %contains_wildcards..ss[GZ1CW,GZs)anyWILDCARD_CHARACTERS)r7s`r&contains_wildcardsrBrs [GZ[ [[r%patterns DataFilesListc [U[5(aGUR5VVs0sH)up[U5[U[5(aUOU/_M+ snn$[U[5(a [ U/0$[U[5(Ga[ SU55(aUHen[U[5(aA[U5S:Xa2SU;a,[URS5[[45(aMY[SU35e UVs/sHo3SPM nn[[U55[U5:wa[SU35eUVs0sH3n[US5[US[5(aUSOUS/_M5 sn$[ U0$[[ U55$s snnfs snfs snf)a Take the data_files patterns from the user, and format them into a dictionary. Each key is the name of the split, and each value is a list of data files patterns (paths or urls). The default split is "train". Returns: patterns: dictionary of split_name -> list of patterns c3B# UHn[U[5v M g7fr;) isinstancedict)r<r7s r&r>$sanitize_patterns..sAWz'4((ssplitpathz]Expected each split to have a 'path' key which can be a string or a list of strings, but got z*Some splits are duplicated in data_files: ) rGrHitemsstrlistSANITIZED_DEFAULT_SPLITr@lenget ValueErrorsetsanitize_patterns)rCkeyvaluer7splitss r&rUrUvs(D!!ZbZhZhZjkZjJCC:eT#:#:%GZjkk Hc " "'(44 Hd # # AA A A#w--G )7*"7;;v#6d DD$wxxAB$7??h7g&hF?3v;3v;. #MfX!VWW ('GGG$%*WV_VZ:[:[wvbijpbqarr'  ,X6 6 h003l@s0F3F9:F>matched_rel_pathc\[U5RRVs/sHo"RS5(dMUPM nn[U5RRVs/sHo"RS5(dMUPM nn[ U5[ U5:g$s snfs snf)u When a path matches a pattern, we additionally check if it's inside a special directory we ignore by default (if it starts with a double underscore). Users can still explicitly request a filepath inside such a directory if "__pycache__" is mentioned explicitly in the requested pattern. Some examples: base directory: ./ └── __pycache__ └── b.txt >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "**") True >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "*/b.txt") True >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__pycache__/*") False >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__*/*") False __)rparentparts startswithrQ)rYr7partdata_dirs_to_ignore_in_pathdata_dirs_to_ignore_in_patterns r&"_is_inside_unrequested_special_dirrbs85==M4N4U4U4[4["u4[D_n_nos_t44["u7?7H7O7O7U7U%o7UtYhYhimYnd7U"%o * +s3Q/R RR#v%osB$B$(B)B)c|[U5RVs/sH.o"RS5(dM[U5S1:XaM,UPM0 nn[U5RVs/sH.o"RS5(dM[U5S1:XaM,UPM0 nn[ U5[ U5:g$s snfs snf)u When a path matches a pattern, we additionally check if it's a hidden file or if it's inside a hidden directory we ignore by default, i.e. if the file name or a parent directory name starts with a dot. Users can still explicitly request a filepath that is hidden or is inside a hidden directory if the hidden part is mentioned explicitly in the requested pattern. Some examples: base directory: ./ └── .hidden_file.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", ".*") False base directory: ./ └── .hidden_dir └── a.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".*/*") False >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".hidden_dir/*") False base directory: ./ └── .hidden_dir └── .hidden_file.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/*") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/.*") False >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/*") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/.*") False .)rr]r^rTrQ)rYr7r_hidden_directories_in_pathhidden_directories_in_patterns r&?_is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dirrgsl""2399"9__S=QZ]^bZchkglZl9""'*00%0OOC4HQTUYQZ_b^cQc0"% ) *c2O.P PP "%s"B4B4B4&B9B9B9pattern_resolverc [GHnURSS5nU"U5n[U5S:dM0[ 5nUH:n[ [ U5[ U55nUceURUS5 M< [SU55(a[S[SUS35e[Vs/sHowU;dM [U5PM sn[U[Vs1sHn[U5iM sn- 5-nUVs0sHowURUS 9/_M sns $ [Hqn /n U R!5H:up{U H/nU"U5n[U5S:dMU R#U5 M8 M< U (dM\U Vs0sHowX_M sns $ [S WS U35e![a GMf=fs snfs snfs snf![a Mf=fs snf) a Get the default pattern from a directory or repository by testing all the supported patterns. The first patterns to return a non-empty list of data files is returned. In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. z{split}*rrKc3b# UH%n[R"[U5(+v M' g7fr;)rematchr)r<rKs r&r>+_get_data_files_patterns..s!Fverxx 5111vs-/zSplit name should match 'z '' but got 'z'.)rKzCouldn't resolve pattern z with resolver )ALL_SPLIT_PATTERNSreplaceFileNotFoundErrorrQrTrraddr@rSrDEFAULT_SPLITSrNsortedformatALL_DEFAULT_PATTERNSrMappend) rh split_patternr7 data_filesrXpp_partsrK sorted_splits patterns_dictnon_empty_splitsrCs r&_get_data_files_patternsrs,+ '' 37 )'2J z?Q "uF(1y7OP*** 77+, FvFFF #>MZ Z%,(. ,224OE#!1'!:Jz?Q&$++E2$ 5  =MN=ME=//=MN N. 7yP`Oab cc;!   XAZ) OsAF36 GG"G G G G%3 GG G" !G" base_pathallowed_extensionsdownload_configc d^[U5(a [X5nOF[U5(a4[RR U5S[R -nOSn[XS9up[U40UD6upV[[5[U51- n[UR[5(a UROURSnUS:waUS-OSn 0n US:XaSU S'UR"U4S S 0U D6R!5V V s/sHupU S S:Xd[U R#S 5(dM&[RR%[RR'U 55(dMi[U 5U;dMz[)X5(aM[+X5(aMU R-U 5(aU OX-PM n n n TbU V s/sH<n [/U4Sj[U 5R1S5SS 55(dM:U PM> nn [3U5[3U 5:a;[5[U 5[U5- 5n[6R9SUSU35 OU nU(d%SUS3nTbUS[5T53- n[;U5eU$s sn n fs sn f)aY Resolve the paths and URLs of the data files from the pattern passed by the user. You can use patterns to resolve multiple local files. Here are a few examples: - *.csv to match all the CSV files at the first level - **.csv to match all the CSV files at any level - data/* to match all the files inside "data" - data/** to match all the files inside "data" and its subdirectories The patterns are resolved using the fsspec glob. In fsspec>=2023.12.0 this is equivalent to Python's glob.glob, Path.glob, Path.match and fnmatch where ** is unsupported with a prefix/suffix other than a forward slash /. More generally: - '*' matches any character except a forward-slash (to match just the file or directory name) - '**' matches any character including a forward-slash / Hidden files and directories (i.e. whose names start with a dot) are ignored, unless they are explicitly requested. The same applies to special directories that start with a double underscore like "__pycache__". You can still include one if the pattern explicitly mentions it: - to include a hidden file: "*/.hidden.txt" or "*/.*" - to include a hidden directory: ".hidden/*" or ".*/*" - to include a special directory: "__special__/*" or "__*/*" Example:: >>> from datasets.data_files import resolve_pattern >>> base_path = "." >>> resolve_pattern("docs/**/*.py", base_path) [/Users/mariosasko/Desktop/projects/datasets/docs/source/_config.py'] Args: pattern (str): Unix pattern or paths or URLs of the data files to resolve. The paths can be absolute or relative to base_path. Remote filesystems using fsspec are supported, e.g. with the hf:// protocol. base_path (str): Base path to use when resolving relative paths. allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions). For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"] download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. Returns: List[str]: List of paths or URLs to the local or remote files that match the patterns. rrfilez://hfF expand_infodetailTtypeislinkNc34># UH nSU-T;v M g7f)rdNr)r<suffixrs r&r>"resolve_pattern..rsgDf&3<#55Dfsrdrz Some files matched the pattern 'z-' but don't have valid data file extensions: zUnable to find ''z with any supported extension )rrrosrL splitdriver6rr rTFILES_TO_IGNORErrGprotocolrNglobrMrRisfilerealpathrbrgr^r@rKrQrOloggerinforq)r7rrrstorage_optionsfs fs_patternfiles_to_ignorerprotocol_prefix glob_kwargsfilepathr matched_pathsoutinvalid_matched_files error_msgs ` r&resolve_patternr's`   + w  GG&&w/2RVV;  @jGw:/:NB/*i.@-AAO(c::r{{ AH*2f*@ggnnRWWM]M]^fMg>h Yx  7 Y38H Y PPXe YH''88o>XXR%* )gIhDWD]D]^aDbcdceDfgg )  s8c-( ($(]);c#h)F$G ! KK27);hi~hA  &wiq1  ) 9$?Q:R9ST TI ** J5 s0%J'] ^_eiijs 6 data_filecR[XS9up[U40UD6tp4[U[5(a)UR U5nUR UR 4$UR[R5(a~[[RURS9nSU[[R5S-SRSSS5-nUR U5nUR UR 4$URU5nSHnX;dM [Xx54s $ g) Nr)endpointtokenhf://rz /resolve/@)ETagetagmtimer)rr rGr resolve_pathrepo_idrevisionr^r HF_ENDPOINTrrQrprrN) rrrr_ resolved_pathhffsrrVs r&_get_single_origin_metadatars"C9!nI y 4O 4FB"l## 2 $$m&<&<<<   f00 1 1V%7%7?T?TUiF,>,>(?!(C(EFNN{\_abcc )))4 $$m&<&<<< 779 D( ; N$ $) r%ry max_workersc :UbUO[Rn[SU55(a8[US[ U5S:*=(d SS9Vs/sH n[ X1S9PM sn$[ [[ US9UU[S[ U5S:*=(d SS9$s snf)Nc3,# UH nSU;v M g7f)rNr)r<rs r&r>'_get_origin_metadata..s <I7i szResolving data files)descdisabler)r tqdm_classrr)r&HF_DATASETS_MULTITHREADING_MAX_WORKERSallhf_tqdmrQrr r)ryrrrs r&_get_origin_metadatars "-!8+f>k>kK < <<< %+J2-5    ( S   +_M #J2%-  sBc^\rSrSrSrS\\S\\SS4U4SjjrSSjr \ SS \\S \ RRS \\S \\\S \\SS4 Sjj5r\ SS \\S \\S \\\S \\SS4 Sjj5r\ SS \\S \\S \\\S \\SS4 Sjj5rSSS.S\\\S\\\SS4SjjrSrU=r$)rDia List of data files (absolute local paths or URLs). It has two construction methods given the user's data files patterns: - ``from_hf_repo``: resolve patterns inside a dataset repository - ``from_local_or_remote``: resolve patterns from a local path Moreover, DataFilesList has an additional attribute ``origin_metadata``. It can store: - the last modified time of local files - ETag of remote files - commit sha of a dataset repository Thanks to this additional attribute, it is possible to hash the list and get a different hash if and only if at least one file changed. This is useful for caching Dataset objects that are obtained from a list of data files. ryorigin_metadatar8Nc0>[TU]U5 X lgr;)super__init__r)selfryr __class__s r&rDataFilesList.__init__#s $.r%cP[/UQUQURUR-5$r;)rDrrothers r&__add__DataFilesList.__add__'s(_t_e_d.B.BUEZEZ.Z[[r%rC dataset_inforrrcSURSURSU=(d S3RS5nURXXES9$)Nzhf://datasets/r/rrrr)idsharstrip from_patterns)clsrCrrrrs r& from_hf_repoDataFilesList.from_hf_repo*sX%\__$5Q|7G7G6H)/WYIZ[bbcfg   >P!  r%cxUbUO%[5R5R5nURXX4S9$Nr)rresolveas_posixr)rrCrrrs r&from_local_or_remote"DataFilesList.from_local_or_remote8s@"+!6IDFNNP!  r%c  UbUO%[5R5R5n/nUHnUR[ UUUUS95 M! [XTS9nU"XW5$![ a [ U5(deMRf=fNrr)rrrextendrrqrr)rrCrrrryr7rs r&rDataFilesList.from_patternsEs"+!6IDFNN'DataFilesList.filter..ds"HZc299S>>Z#%z.*(z )(\..+)?$c3N# UHn[R"U5v M g7fr;r)r<fns r&r>rgs!E*B"))B--*rz.*[\/]?(z)$c3D># UHoRT5v M g7fr;)rm)r<r7rs r&r>rks7i`hU\ i8P8P`hs r)joinrwrlcompilerDr@rrO)rrrrC ext_pattern fn_patternrs `r&filterDataFilesList.filter_s (("HZ"HHK OOBJJ[M'DE F !E*!EEJ OOBJJ(:,b'AB C  ,0jDyC7i`h7i4iDj $ 4 4  !dT=Q=QR R ks %C:C:r)rrDr8rDNNN)r r!r"r#__doc__rOrNSingleOriginMetadatarr classmethodhuggingface_hubhf_api DatasetInforrrrrrr$ __classcell__rs@r&rDrDs"/49/tDX?Y/^b/\ $(2648  s)  &,,88  C=  %T#Y/  ".1       $(2648  s)  C=  %T#Y/  ".1      $(2648 0s)0C=0%T#Y/ 0 ".1 0  00448[_S%d3i0SEMdSViEXS SSr%c\rSrSrSr\SS\\\\ \\ 44S\ \S\ \ \S\ \ SS4 S jj5r \SS\\\\ \\ 44S \RR S\ \S\ \ \S\ \ SS4 S jj5r\SS\\\\ \\ 44S\ \S\ \ \S\ \ SS4 S jj5rSSS .S\ \ \S\ \ \SS4SjjrSrg) DataFilesDictiras Dict of split_name -> list of data files (absolute local paths or URLs). It has two construction methods given the user's data files patterns : - ``from_hf_repo``: resolve patterns inside a dataset repository - ``from_local_or_remote``: resolve patterns from a local path Moreover, each list is a DataFilesList. It is possible to hash the dictionary and get a different hash if and only if at least one file changed. For more info, see [`DataFilesList`]. This is useful for caching Dataset objects that are obtained from a list of data files. Changing the order of the keys of this dictionary also doesn't change its hash. NrCrrrr8cU"5nUR5H4upg[U[5(aUO[RUUUUS9XV'M6 U$r)rMrGrDrrrCrrrrrVpatterns_for_keys r&r"DataFilesDict.from_local_or_remotesae%-^^%5 !C. >>!"77$''9$3 8 H&6 r%rc U"5nUR5H5upx[U[5(aUO[RUUUUUS9Xg'M7 U$)N)rrrr)rMrGrDr) rrCrrrrrrVr s r&rDataFilesDict.from_hf_reposde%-^^%5 !C. >>!"//$!-''9$3 0 H&6 r%cU"5nUR5H4upg[U[5(aUO[RUUUUS9XV'M6 U$r)rMrGrDrrs r&rDataFilesDict.from_patternssae%-^^%5 !C. >>!"00$''9$3 1 H&6 r%rrrcz[U5"5nUR5HupEURXS9X4'M U$)Nr)rrMr)rrrrrVdata_files_lists r&rDataFilesDict.filters<4jl$(JJL C&---[CH%1 r%rr)r r!r"r#rrrHrNr rOrDrrrrrrrrrr$rr%r&rrrs $(2648 sE$s)]":;;<C=%T#Y/  ".1   * $(2648 sE$s)]":;;<&,,88C=  %T#Y/  ".1  .$(2648 sE$s)]":;;<C=%T#Y/  ".1   ,48[_%d3i0EMdSViEX r%rc^\rSrSrSrS\\S\\\\4U4SjjrSr \ SS\\S\\\SS4Sjj5r SS \S \\ SS 4S jjr S \\SS4SjrSrU=r$)DataFilesPatternsListiz List of data files patterns (absolute local paths or URLs). For each pattern there should also be a list of allowed extensions to keep, or a None ot keep all the files for the pattern. rCrc0>[TU]U5 X lgr;)rrr)rrCrrs r&rDataFilesPatternsList.__init__s ""4r%cP[/UQUQURUR-5$r;)rDrrs r&rDataFilesPatternsList.__add__s(_t_e_d.E.EH`H`.`aar%r8c,U"X/[U5-5$r;)rQ)rrCrs r&r#DataFilesPatternsList.from_patternss81CMABBr%rrrDc <UbUO%[5R5R5n/n[XR5H!upEUR [ UUUUS95 M# [X2S9n[X65$![a [U5(deMWf=fr) rrrziprrrrqrrrD)rrrryr7rrs r&rDataFilesPatternsList.resolves "+!6IDFNNtCy6IC CC59::".1:  :. DI :Q  r%rc\rSrSrSr\SS\\\\4S\ \\SS4Sjj5r SS\S \ \ SS 4S jjr S \\SS4S jr Srg)DataFilesPatternsDictizS Dict of split_name -> list of data files patterns (absolute local paths or URLs). NrCrr8cU"5nUR5H2upE[U[5(aUO[RUUS9X4'M4 U$)Nr )rMrGrr)rrCrrrVr s r&r#DataFilesPatternsDict.from_patterns s\e%-^^%5 !C.0EFF!*88$'99 H&6 r%rrrcr[5nUR5HupEURX5X4'M U$r;)rrMr)rrrrrVdata_files_patterns_lists r&rDataFilesPatternsDict.resolves5 o-1ZZ\ )C/77 SCH.: r%rc~[U5"5nUR5Hup4URU5X#'M U$r;)rrMr)rrrrVr&s r&r'DataFilesPatternsDict.filter_extensions#s74jl-1ZZ\ )C/AA*MCH.: r%rr;)r r!r"r#rrrHrNrOrrrrrr$rr%r&r"r"sW[ CcN+ AI$s)AT    $59".1  DI:Qr%r")NNr;)Wrrl functoolsrrrpathlibrrtypingrrr r fsspec.corer r packagingr tqdm.contrib.concurrentr rrdownloadrnamingrrXrutilsrrrutils.file_utilsrrrrrutils.py_utilsrtuplerNrTRAINrP get_loggerr rrrqr)SPLIT_PATTERN_SHARDED VALIDATIONTESTSPLIT_KEYWORDSNON_WORDS_CHARSFSSPEC_VERSIONparse"KEYWORDS_IN_FILENAME_BASE_PATTERNS"KEYWORDS_IN_DIR_NAME_BASE_PATTERNSrsru"DEFAULT_PATTERNS_SPLIT_IN_FILENAME"DEFAULT_PATTERNS_SPLIT_IN_DIR_NAMEDEFAULT_PATTERNS_ALLrorvrArboolrBrHrOrUrbrgrrrrintrrDrrr")rKr5r7s000r&rFs ",,!(.$"rr*U38_eCj%)CDekk*   H % #  ) a KK':& ; JJ9  7==44*GI\)]&*&  W]];77*IK^)_&*&+AB`)a&*&++u//< &    %e, ,G9G wO<9 =,   &" &    %e, ,G9G wO<9 =,   &" KK$,,&& \\\#1dD#o 6#14U4PS9VeKeEf@f;g#1LSSsStSB;QVY;Qdg;Qlp;Q|)dxtCy8H/I)ddSVX\]`XaSaNb)d^/304 Y YY!c+Yn- Y  #Y YxXjXjx7OXj[_`ceijmen`n[oXjz15n-015!%S n-#  :^SDI^SBZDm+,Zz2 DI2 j#D&;!;<#q & &s0;L $L.LL($L"4L(L"L(