
    h                     b    S SK Jr  S SKrSSKJrJr  SSKJr  SSKJ	r	  SSK
Jr   " S	 S
\5      rg)    )OptionalN   )Features
NamedSplit)DownloadMode)Spark   )AbstractDatasetReaderc                      ^  \ rS rSrSr        SS\R                  R                  S\\	   S\\
   S\S\S\S	\S
\S\4U 4S jjjrS rSrU =r$ )SparkDatasetReader   zA dataset reader that reads from a Spark DataFrame.

When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be
provided. Streaming is not currently supported.
dfsplitfeatures	streaming	cache_dirkeep_in_memoryworking_dirload_from_cache_filefile_formatc
           	      t   > [         TU ]  " SUUUUUS.U
D6  Xl        Xl        [	        SUUUUS.U
D6U l        g )N)r   r   r   r   r   )r   r   r   r    )super__init___load_from_cache_file_file_formatr   builder)selfr   r   r   r   r   r   r   r   r   kwargs	__class__s              K/home/james-whalen/.local/lib/python3.13/site-packages/datasets/io/spark.pyr   SparkDatasetReader.__init__   se     	 	
)	
 	
 &:"' 
#	

 
    c                 >   U R                   (       a#  U R                  R                  U R                  S9$ U R                  (       a  S O[
        R                  nU R                  R                  UU R                  S9  U R                  R                  U R                  S9$ )N)r   )download_moder   )
r   r   as_streaming_datasetr   r   r   FORCE_REDOWNLOADdownload_and_preparer   
as_dataset)r   r%   s     r!   readSparkDatasetReader.read1   s{    >><<444::4FF $ : :@]@]))')) 	* 	
 ||&&TZZ&88r#   )r   r   r   )NNTNFNTarrow)__name__
__module____qualname____firstlineno____doc__pysparksql	DataFramer   r   r   boolstrr   r*   __static_attributes____classcell__)r    s   @r!   r   r      s     '+'+$%)"
KK!!
 
#
 8$	

 
 
 
 
 #
 
 
>9 9r#   r   )typingr   r2    r   r   downloadr   packaged_modules.spark.sparkr   abcr
   r   r   r#   r!   <module>r>      s$      # # 0 &.9. .9r#   