
    h                         S SK r S SKJr  S SKJrJr  SSKJr  SSKJ	r	J
r
JrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJr  SSK J!r!  SSK"J#r#  \!" \$5      r%\(       a  SS	K&J'r'  SS
\\   4S jjr(SS jr)g)    N)wraps)TYPE_CHECKINGOptional   )DownloadConfig)	xbasenamexdirname	xet_parsexexistsxgetsizexglob
xgzip_openxisdirxisfilexjoinxlistdirxnumpy_loadxopenxpandas_read_csvxpandas_read_excelxPathxpyarrow_parquet_read_tablexrelpathxsio_loadmatxsplit	xsplitextxwalkxxml_dom_minidom_parse)
get_logger)patch_submoduleDatasetBuilderdownload_configc                   ^ [         R                  " U 5      n[        US5      (       ag  UR                  (       aV  [	        UR                  [
        5      (       a6  TR                  UR                  l        TR                  UR                  l        gU4S jn[        USU" [        5      5      R                  5         [        USU" [        5      5      R                  5         [        USU" [        5      5      R                  5         [        USU" [        5      5      R                  5         [        US[        5      R                  5         [        US	[        5      R                  5         [        US
[         5      R                  5         [        US["        5      R                  5         [        US[$        5      R                  5         [        US[&        5      R                  5         [        USU" [(        5      5      R                  5         [        USU" [*        5      5      R                  5         [        USU" [,        5      5      R                  5         [        USU" [.        5      5      R                  5         [        US[0        5      R                  5         [        USU" [2        5      5      R                  5         [        USU" [4        5      5      R                  5         [        USU" [6        5      S/S9R                  5         [        USU" [8        5      S/S9R                  5         [        USU" [:        5      S/S9R                  5         [        USU" [<        5      5      R                  5         [        USU" [>        5      5      R                  5         UR@                  RC                  S5      (       d%  [        USU" [D        5      5      R                  5         TUl        g)a  Extend the module to support streaming.

We patch some functions in the module to use `fsspec` to support data streaming:
- We use `fsspec.open` to open and read remote files. We patch the module function:
  - `open`
- We use the "::" hop separator to join paths and navigate remote compressed/archive files. We patch the module
  functions:
  - `os.path.join`
  - `pathlib.Path.joinpath` and `pathlib.Path.__truediv__` (called when using the "/" operator)

The patched functions are replaced with custom functions defined to work with the
:class:`~download.streaming_download_manager.StreamingDownloadManager`.

Args:
    module_path: Path to the module to be extended.
    download_config: Mainly use `token` or `storage_options` to support different platforms and auth types.
_patched_for_streamingNc                 B   >^  [        T 5      UU 4S j5       nSUl        U$ )Nc                     > T" U ST0UD6$ )Nr#    )argskwargsr#   functions     L/home/james-whalen/.local/lib/python3.13/site-packages/datasets/streaming.pywrapper?extend_module_for_streaming.<locals>.wrap_auth.<locals>.wrapperG   s    TM?MfMM    	wrap_auth)r   _decorator_name_)r+   r-   r#   s   ` r,   r0   .extend_module_for_streaming.<locals>.wrap_authF   s*    	x	N 
	N $/ r/   openz
os.listdirzos.walkz	glob.globzos.path.joinzos.path.dirnamezos.path.basenamezos.path.relpathzos.path.splitzos.path.splitextzos.path.existszos.path.isdirzos.path.isfilezos.path.getsizezpathlib.Pathz	gzip.openz
numpy.loadzpandas.read_csv__version__)attrszpandas.read_excelzscipy.io.loadmatzxml.etree.ElementTree.parsezxml.dom.minidom.parsezdatasets.packaged_modules.zpyarrow.parquet.read_table)#	importlibimport_modulehasattrr%   
isinstancer   tokenstorage_optionsr    r   startr   r   r   r   r	   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r
   r   __name__
startswithr   )module_pathr#   moduler0   s    `  r,   extend_module_for_streamingrA   *   s   & $$[1F v/00V5R5Rf33^DD2A2G2GF))/<K<[<[F))9 FFIe$45;;=FL)H*=>DDFFIy'78>>@FK5)9:@@BFNE288:F-x8>>@F.	:@@BF-x8>>@FOV4::<F.	:@@BF,i.@AGGIFOYv->?EEGF,i.@AGGIF-y/BCIIKFNE288:FK:)>?EEGFL)K*@AGGIF-y9I/JS`RabhhjF/;M1NWdVefllnF.	,0GP]_eegF99Y;OPVVXF3Y?U5VW]]_??%%&BCC <iHc>dekkm$3F!r/   c                 b   [        U R                  U R                  S9n[        U R                  US9  SSKJn  [        U 5      R                  SS  Vs/ s H=  n[        X25      (       d  M  UR                  UR                  :w  d  M1  UR                  PM?     nnU H  n[        XQS9  M     gs  snf )zExtend the dataset builder module and the modules imported by it to support streaming.

Args:
    builder (:class:`DatasetBuilder`): Dataset builder instance.
)r;   r:   )r#   r   r!   N)
r   r;   r:   rA   
__module__builderr"   type__mro__
issubclass)rD   r#   r"   clsparent_builder_modulesr@   s         r,   $extend_dataset_builder_for_streamingrJ   n   s     %W5L5LT[TaTabO 2 2OT
 ( =((,,Cc* 	/2~~AZAZ/Z 	,  
 )#FL )s   B,+B,B,)N)rD   r"   )*r6   	functoolsr   typingr   r   download.download_configr   utils.file_utilsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   utils.loggingr   utils.patchingr    r=   loggerrD   r"   rA   rJ   r(   r/   r,   <module>rR      sj      * 4      2 & + 
H	 'A4h~>V A4HMr/   