
    h)                         S SK r S SKrS SKJrJrJr  S SKrSSKJrJ	r	J
r
Jr  SSKJr  SSKJr  SSKJr  SSKJrJr  S	S
KJr   " S S\5      r " S S5      rg)    N)BinaryIOOptionalUnion   )DatasetFeatures
NamedSplitconfig)query_table)Json)tqdm)NestedDataStructureLikePathLike   )AbstractDatasetReaderc                      ^  \ rS rSr       SS\\   S\\   S\\   S\	S\
S\
S\\	   S	\\   4U 4S
 jjjrS rSrU =r$ )JsonDatasetReader   path_or_pathssplitfeatures	cache_dirkeep_in_memory	streamingfieldnum_procc	           
         > [         T
U ]  " U4UUUUUUS.U	D6  Xpl        [        U[        5      (       a  UOU R
                  U0n[        SUUUUS.U	D6U l        g )N)r   r   r   r   r   r   )r   
data_filesr   r    )super__init__r   
isinstancedictr   r   builder)selfr   r   r   r   r   r   r   r   kwargs	__class__s             J/home/james-whalen/.local/lib/python3.13/site-packages/datasets/io/json.pyr!   JsonDatasetReader.__init__   s     			
)		
 		
 
)3M4)H)Htzz[hNi 
$	

 
    c                 ,   U R                   (       a%  U R                  R                  U R                  S9nU$ S nS nS nS nU R                  R	                  UUUUU R
                  S9  U R                  R                  U R                  X@R                  S9nU$ )N)r   )download_configdownload_modeverification_mode	base_pathr   )r   r.   	in_memory)r   r$   as_streaming_datasetr   download_and_preparer   
as_datasetr   )r%   datasetr,   r-   r.   r/   s         r(   readJsonDatasetReader.read0   s    >>ll77djj7IG$  #O M $ILL-- /+"3# .  ll--jj4EQdQd . G r*   )r$   r   )NNNFFNN)__name__
__module____qualname____firstlineno__r   r   r   r	   r   strboolintr!   r5   __static_attributes____classcell__)r'   s   @r(   r   r      s     '+'+$#"&
.x8
 
#
 8$	

 
 
 
 }
 3-
 
@ r*   r   c                   ~    \ rS rSr   SS\S\\\4   S\\	   S\\	   S\\
   4
S jjrS	\	4S
 jrS rS\S	\	4S jrSrg)JsonDatasetWriterH   Nr4   path_or_buf
batch_sizer   storage_optionsc                     Ub  US::  a  [        SU S35      eXl        X l        U(       a  UO[        R                  U l        X@l        SU l        U=(       d    0 U l        X`l	        g )Nr   z	num_proc z must be an integer > 0.zutf-8)

ValueErrorr4   rC   r
   DEFAULT_MAX_BATCH_SIZErD   r   encodingrE   to_json_kwargs)r%   r4   rC   rD   r   rE   rJ   s          r(   r!   JsonDatasetWriter.__init__I   s`     HMy
2JKLL&(2*8U8U .4",r*   returnc                    U R                   R                  SS 5      nU R                   R                  SS5      nU R                   R                  SUS:X  a  SOS5      nSU R                   ;  a  US;   a  SU R                   S'   [        U R                  [        [
        [        R                  45      (       a  S	OS nU R                   R                  S
U5      nUS;  a  [        SU S35      eU(       d/  U R                  U R                  R                  :  a  [        S5      e[        U R                  [        [
        [        R                  45      (       aa  [        R                  " U R                  S4S
U0U R                  =(       d    0 D6 nU R                  " SXbUS.U R                   D6nS S S 5        U$ U(       a  [        SU S35      eU R                  " SU R                  X#S.U R                   D6nU$ ! , (       d  f       W$ = f)NrC   orientrecordslinesTFindex)r   tableinfercompression)NrS   gzipbz2xzz&`datasets` currently does not support z compressionzOutput JSON will not be formatted correctly when lines = False and batch_size < number of rows in the dataset. Use pandas.DataFrame.to_json() instead.wb)file_objrN   rP   zUThe compression parameter is not supported when writing to a buffer, but compression=z1 was passed. Please provide a local path instead.r   )rJ   popr"   rC   r;   bytesosr   NotImplementedErrorrD   r4   num_rowsfsspecopenrE   _write)r%   _rN   rP   default_compressionrT   bufferwrittens           r(   writeJsonDatasetWriter.write]   s   ##M48$$((9=##''99LRWX$---&<N2N+0D( *4D4D4DsESUS^S^F_)`)`gfj))--m=PQBB%(N{m[g&hii4<<+@+@@% i  d&&eR[[(ABB  $4?DHDXDXD^\^++ivEiUYUhUhi  )klwkxGH  kko4+;+;Fo[_[n[noG  s    G77
Hc                 P   Uu  p#pE[        U R                  R                  [        X"U R                  -   5      U R                  R
                  S9nUR                  5       R                  " SS X4S.UD6nUR                  S5      (       d  US-  nUR                  U R                  5      $ )N)rR   keyindices)rC   rN   rP   
r   )r   r4   dataslicerD   _indices	to_pandasto_jsonendswithencoderI   )r%   argsoffsetrN   rP   rJ   batchjson_strs           r(   _batch_jsonJsonDatasetWriter._batch_json~   s    04-,,##ft67LL))

 ??$,,lfl]kl  &&Ht}}--r*   rY   c                    SnU R                   b  U R                   S:X  ab  [        [        S[        U R                  5      U R
                  5      SSS9 H)  nU R                  XbX445      nXQR                  U5      -  nM+     U$ [        U R                  5      U R
                  p[        R                  " U R                   5       n
[        U
R                  U R                  [        SX5       Vs/ s H  ofX#U4PM	     sn5      X-  (       a  X-  S-   OX-  SSS9 H  nXQR                  U5      -  nM     SSS5        U$ s  snf ! , (       d  f       U$ = f)z{Writes the pyarrow table as JSON lines to a binary file handle.

Caller is responsible for opening and closing the handle.
r   Nr   bazCreating json from Arrow format)unitdesc)totalr{   r|   )r   hf_tqdmrangelenr4   rD   rw   rf   multiprocessingPoolimap)r%   rY   rN   rP   rJ   re   rt   rv   r^   rD   pools              r(   ra   JsonDatasetWriter._write   s?    == DMMQ$6!aT\\*DOO<6
  ++VU,ST>>(33*  $'t||#4dooj %%dmm4 'II((OTUVX`OmnOmV&@Omn ;C:O81Q6U]Uk:!H ~~h77G! 5  o	 54 s   *E,D<:8E<E
E)rD   r4   rI   r   rC   rE   rJ   )NNN)r7   r8   r9   r:   r   r   r   r   r   r=   r#   r!   rf   rw   ra   r>   r   r*   r(   rA   rA   H   s    
 %)"&*.-- 8X-.- SM	-
 3-- "$-(s B.## 
#r*   rA   )r   r\   typingr   r   r   r_    r   r   r	   r
   
formattingr   packaged_modules.json.jsonr   utilsr   r~   utils.typingr   r   abcr   r   rA   r   r*   r(   <module>r      sC     	 , ,  4 4 $ - # < &6- 6rf fr*   