
    h                         S SK r S SKrS SKJrJrJr  S SKrSSKJrJ	r	J
r
Jr  SSKJr  SSKJr  SSKJr  SSKJrJr  S	S
KJr   " S S\5      r " S S5      rg)    N)BinaryIOOptionalUnion   )DatasetFeatures
NamedSplitconfig)query_table)Csv)tqdm)NestedDataStructureLikePathLike   )AbstractDatasetReaderc                   v   ^  \ rS rSr      SS\\   S\\   S\\   S\	S\
S\
S\\   4U 4S	 jjjrS
 rSrU =r$ )CsvDatasetReader   path_or_pathssplitfeatures	cache_dirkeep_in_memory	streamingnum_procc           
         > [         T	U ]  " U4UUUUUUS.UD6  [        U[        5      (       a  UOU R                  U0n[        SUUUS.UD6U l        g )N)r   r   r   r   r   r   )r   
data_filesr    )super__init__
isinstancedictr   r   builder)
selfr   r   r   r   r   r   r   kwargs	__class__s
            I/home/james-whalen/.local/lib/python3.13/site-packages/datasets/io/csv.pyr    CsvDatasetReader.__init__   sy     			
)		
 		
 *4M4)H)Htzz[hNi 
$
 	
    c                 ,   U R                   (       a%  U R                  R                  U R                  S9nU$ S nS nS nS nU R                  R	                  UUUUU R
                  S9  U R                  R                  U R                  X@R                  S9nU$ )N)r   )download_configdownload_modeverification_mode	base_pathr   )r   r-   	in_memory)r   r#   as_streaming_datasetr   download_and_preparer   
as_datasetr   )r$   datasetr+   r,   r-   r.   s         r'   readCsvDatasetReader.read-   s    >>ll77djj7IG$  #O M $ILL-- /+"3# .  ll--jj4EQdQd . G r)   )r#   )NNNFFN)__name__
__module____qualname____firstlineno__r   r   r   r	   r   strboolintr    r4   __static_attributes____classcell__)r&   s   @r'   r   r      s     '+'+$"&
.x8
 
#
 8$	

 
 
 
 3-
 
: r)   r   c                   ~    \ rS rSr   SS\S\\\4   S\\	   S\\	   S\\
   4
S jjrS	\	4S
 jrS rS\S	\	4S jrSrg)CsvDatasetWriterE   Nr3   path_or_buf
batch_sizer   storage_optionsc                     Ub  US::  a  [        SU S35      eXl        X l        U(       a  UO[        R                  U l        X@l        SU l        U=(       d    0 U l        X`l	        g )Nr   z	num_proc z must be an integer > 0.zutf-8)

ValueErrorr3   rB   r
   DEFAULT_MAX_BATCH_SIZErC   r   encodingrD   to_csv_kwargs)r$   r3   rB   rC   r   rD   rI   s          r'   r    CsvDatasetWriter.__init__F   s`     HMy
2JKLL&(2*8U8U .4"*r)   returnc                 J   U R                   R                  SS 5      nU R                   R                  SS5      nU R                   R                  SS5      n[        U R                  [        [
        [        R                  45      (       a_  [        R                  " U R                  S40 U R                  =(       d    0 D6 nU R                  " SXBUS.U R                   D6nS S S 5        U$ U R                  " SU R                  X#S.U R                   D6nU$ ! , (       d  f       W$ = f)	NrB   headerTindexFwb)file_objrM   rN   r   )rI   popr!   rB   r:   bytesosr   fsspecopenrD   _write)r$   _rM   rN   bufferwrittens         r'   writeCsvDatasetWriter.writeZ   s    ""=$7##''$7""&&w6d&&eR[[(ABBT--tT8L8L8RPRTX^++hvEhUYUgUgh U  kkn4+;+;Fn[_[m[mnG	 UT s   > D
D"c                 ,   Uu  p#pE[        U R                  R                  [        X"U R                  -   5      U R                  R
                  S9nUR                  5       R                  " SS US:X  a  UOSUS.UD6nUR                  U R                  5      $ )N)tablekeyindicesr   F)rB   rM   rN   r   )
r   r3   dataslicerC   _indices	to_pandasto_csvencoderH   )r$   argsoffsetrM   rN   rI   batchcsv_strs           r'   
_batch_csvCsvDatasetWriter._batch_csvf   s    /3,,,##ft67LL))

 //#** 
!V%u
Xe
 ~~dmm,,r)   rP   c                    SnU R                   b  U R                   S:X  ab  [        [        S[        U R                  5      U R
                  5      SSS9 H)  nU R                  XbX445      nXQR                  U5      -  nM+     U$ [        U R                  5      U R
                  p[        R                  " U R                   5       n
[        U
R                  U R                  [        SX5       Vs/ s H  ofX#U4PM	     sn5      X-  (       a  X-  S-   OX-  SSS9 H  nXQR                  U5      -  nM     SSS5        U$ s  snf ! , (       d  f       U$ = f)ztWrites the pyarrow table as CSV to a binary file handle.

Caller is responsible for opening and closing the handle.
r   Nr   bazCreating CSV from Arrow format)unitdesc)totalrn   ro   )r   hf_tqdmrangelenr3   rC   rj   rZ   multiprocessingPoolimap)r$   rP   rM   rN   rI   rY   rg   ri   num_rowsrC   pools              r'   rV   CsvDatasetWriter._writes   s;   
 == DMMQ$6!aT\\*DOO<5
 //65*PQ>>'22,  $'t||#4dooj %%dmm4&IINSTUW_NlmNlF&?Nlm ;C:O81Q6U]Uk9 G ~~g66G  5  n	 54 s   *E,D<:8E<E
E)rC   r3   rH   r   rB   rD   rI   )NNN)r6   r7   r8   r9   r   r   r   r   r   r<   r"   r    rZ   rj   rV   r=   r   r)   r'   r@   r@   E   s|    
 %)"&*.++ 8X-.+ SM	+
 3-+ "$+(
s 
-x C r)   r@   )rt   rS   typingr   r   r   rT    r   r   r	   r
   
formattingr   packaged_modules.csv.csvr   utilsr   rq   utils.typingr   r   abcr   r   r@   r   r)   r'   <module>r      sC     	 , ,  4 4 $ * # < &3, 3lL Lr)   