
    h$                     *   S SK r S SKrS SKJr  S SKJr  S SKrS SKr	S SK
Jr  S SKrS SKrS SKJr  S SKJr  \R&                  R(                  R+                  \5      rS rS rS r\ " S	 S
\R6                  5      5       r " S S\R:                  5      rg)    N)	dataclass)Optional)
table_cast)readlinec                       [         R                  R                  R                  " U 0 UD6$ ! [         a-    [         R                  R                  R
                  " U 0 UD6s $ f = fN)pdiojsonujson_dumpsAttributeErrordumpsargskwargss     ]/home/james-whalen/.local/lib/python3.13/site-packages/datasets/packaged_modules/json/json.pyr   r      R    1uuzz%%t6v66 1uuzz0001   ), 4A#"A#c                       [         R                  R                  R                  " U 0 UD6$ ! [         a-    [         R                  R                  R
                  " U 0 UD6s $ f = fr   )r	   r
   r   ujson_loadsr   loadsr   s     r   r   r      r   r   c                     [         R                  R                  R                  S:  a  SUS'   [        R
                  " U 40 UD6$ )N   pyarrowdtype_backend)datasetsconfigPANDAS_VERSIONmajorr	   	read_json)path_or_bufr   s     r   pandas_read_jsonr"   #   s8    %%++q0"+<<.v..    c                      ^  \ rS rSr% SrSr\\R                     \	S'   Sr
\\	S'   Sr\\   \	S'   Sr\\   \	S'   S	r\\	S
'   Sr\\   \	S'   Sr\\	S'   Sr\\   \	S'   U 4S jrSrU =r$ )
JsonConfig)   zBuilderConfig for JSON.Nfeaturesutf-8encodingencoding_errorsfieldTuse_threads
block_sizei   	chunksizenewlines_in_valuesc                 "   > [         TU ]  5         g r   )super__post_init__)self	__class__s    r   r2   JsonConfig.__post_init__6   s    r#    )__name__
__module____qualname____firstlineno____doc__r'   r   r   Features__annotations__r)   strr*   r+   r,   boolr-   intr.   r/   r2   __static_attributes____classcell__)r4   s   @r   r%   r%   )   s|    !,0Hhx(()0Hc%)OXc])E8C=K $J$Is)--   r#   r%   c                   d    \ rS rSr\rS rS rS\R                  S\R                  4S jr
S rSrg	)
Json:   c                    U R                   R                  b:  [        R                  S5        U R                   R                  U R                   l        U R                   R
                  SLa  [        R                  S5        U R                   R                  b  [        S5      e[        R                  " U R                   R                  S9$ )NzTThe JSON loader parameter `block_size` is deprecated. Please use `chunksize` insteadTzZThe JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore.zEThe JSON loader parameter `newlines_in_values` is no longer supported)r'   )r   r-   loggerwarningr.   r,   r/   
ValueErrorr   DatasetInfor'   )r3   s    r   _info
Json._info=   s    ;;!!-NNqr$(KK$:$:DKK!;;""$.NNl ;;))5dee##T[[-A-ABBr#   c                    U R                   R                  (       d"  [        SU R                   R                   35      eSUR                  l        UR                  U R                   R                  5      n/ nUR                  5        Hc  u  pE[        U[        5      (       a  U/nU Vs/ s H  oaR                  U5      PM     nnUR                  [        R                  " USU0S95        Me     U$ s  snf )z-We handle string, list and dicts in datafilesz=At least one data file must be specified, but got data_files=Tfiles)name
gen_kwargs)r   
data_filesrI   download_configextract_on_the_flydownload_and_extractitems
isinstancer>   
iter_filesappendr   SplitGenerator)r3   
dl_managerrQ   splits
split_namerN   files          r   _split_generatorsJson._split_generatorsI   s    {{%%\]a]h]h]s]s\tuvv8<
""544T[[5K5KL
!+!1!1!3J%%%=BCUT**40UECMM(11zwX]N^_`	 "4
  Ds   'C/pa_tablereturnc           
         U R                   R                  Gb  [        U R                   R                  5      [        UR                  5      -
   Hn  nU R                   R                  R                  R                  U5      R                  nUR                  U[        R                  " S /[        U5      -  US95      nMp     [        UR                  5       H  u  pB[        R                  R                  X   R                  5      (       d  M7  U R                   R                  R                  US 5      [        R                   " S5      :X  d  Mw  X   R#                  [$        R&                  S9R)                  SSS9n[        R                  " S SU-   R+                  S	5       5       [        R,                  " 5       S9nUR/                  XBU5      nM     [1        XR                   R                  R                  5      nU$ )
N)typestring)types_mapperrecordsT)orientlinesc              3   X   #    U  H   o(       d  M  S UR                  5       -   v   M"     g7f){N)rstrip).0xs     r   	<genexpr>#Json._cast_table.<locals>.<genexpr>h   s#     T3NaRS)qxxz)3Ns   
**
z
{)r   r'   setcolumn_namesarrow_schemar+   rc   append_columnpaarraylen	enumeratetypes	is_structgetr   Value	to_pandasr	   
ArrowDtypeto_jsonsplitrd   
set_columnr   )r3   r`   column_namerc   ijsonlstring_arrays          r   _cast_tableJson._cast_tableW   s   ;;+"4;;#7#783x?T?T;UU{{++88>>{KPP#11+rxxQTU]Q^H^ei?jk  V #,H,A,A"B88%%h&;&@&@AAdkkFZFZF^F^G^^H-G. !-"> 	> 
 $&88TD5L3G3G3NT[][d[d[f$L  (221<PH #C "(KK,@,@,M,MNHr#   c              #   
  #    [        [        R                  R                  U5      5       GH  u  p#U R                  R
                  Gb+  [        X0R                  R                  U R                  R                  S9 n[        UR                  5       5      nS S S 5        WU R                  R
                     n[        [        R                  " [        U5      5      5      nUR                  R!                  5       S/:X  aB  U R                  R"                  (       a  [%        U R                  R"                  5      OS/Ul        [&        R(                  R+                  USS9nX R-                  U5      4v   GMI  [        US5       nSn[/        U R                  R0                  S-  S5      n	U R                  R                  b  U R                  R                  OS	n
 UR                  U R                  R0                  5      nU(       d  O XR3                  5       -  nU R                  R                  S
:w  a3  UR9                  U R                  R                  U
S9R;                  S
5      n   [<        R>                  " [        R@                  " U5      [<        RB                  " U	S9S9n X(4U R-                  U5      4v   US-  nM  S S S 5        GM     g ! , (       d  f       GN?= f! [4        [        R6                  4 a    U[3        U5      -  n Nf = f! [&        RD                  [&        RF                  4 ax  n[I        U[&        RD                  5      (       a  S[K        U5      ;  d  U	[M        U5      :  a  e [N        RQ                  S[M        U5       SU	 SU	S-   S35        U	S-  n	 S nAOS nAff = fGMJ  ! [&        RD                   Ga  n [        X0R                  R                  U R                  R                  S9 n[        U5      nS S S 5        OH! , (       d  f       O:= f! [R         a*    [N        RU                  SU S[W        U5       SU 35        Uef = fWR                  R!                  5       S/:X  aB  U R                  R"                  (       a  [%        U R                  R"                  5      OS/Ul         [&        R(                  R+                  USS9nOS! [&        RD                   a<  n[N        RU                  SU S[W        U5       SU 35        [S        SU S35      S eS nAff = fX R-                  U5      4v    S nAGM  S nAff = f! , (       d  f       GM0  = f7f)N)r)   errorsr   textF)preserve_indexrb    i @  strictr(   )r   )r-   )read_options
straddlingz	Batch of z* bytes couldn't be parsed with block_size=z. Retrying with block_size=r   .zFailed to load JSON from file 'z' with error z: z=Failed to convert pandas DataFrame to Arrow Table from file 'z<Failed to convert pandas DataFrame to Arrow Table from file    ),rx   	itertoolschainfrom_iterabler   r+   openr)   r*   r   readr"   r
   StringIOr   columnstolistr'   listru   Tablefrom_pandasr   maxr.   r   r   UnsupportedOperationdecodeencodepajr    BytesIOReadOptionsArrowInvalidArrowNotImplementedErrorrV   r>   rw   rG   debugrI   errorrc   )r3   rN   file_idxr]   fdatasetdfr`   	batch_idxr-   r*   batches                r   _generate_tablesJson._generate_tablesp   s    '	(E(Ee(LMNH{{  ,$)=)=dkkFaFabfg)!&&(3G c "$++"3"34%bkk+g2F&GH::$$&1#-?C{{?S?Sdkk&:&:!;Z`YaBJ88//5/I 0 0 ::: $% !I "%T[[%:%:b%@(!KJ7;{{7R7R7^33dl $  !t{{'<'< =$!1!ZZ\1E  ;;//7:$)LL1E1EoL$^$e$efm$nE*""&!8/2}}(*

5(9cmHn0&H %*J  (3T5E5Eh5OOO!Q	q  &% N cb6 !/0G0GH 1!Xa[0E1 )+9T9T'U !8(21boo(F(F,8A,F+5E
+B(- )/.7E
|Cmnxmy  zU  V`  cd  Vd  Ue  ef  -g)* )3a
!8 #'(  " "(%)$(;;3G3GPTP[P[PkPk&"%&)9!)<B&" &" &" $. ( &/NtfTabfghbiajjlmnlo-p q&'(  "zz002qc9KO;;K_K_T$++2F2F-Gflem
,+-88+?+?SX+?+Y#%?? , &&cdhciivw{|}w~v  @B  CD  BE  %F!" '1&bcgbhhi$j'"'+!,	, #+,<,<X,F"FF!+"S &%s  A9U;KC+U BUKAU$N/&;L!U?U
K	U(L
U	L

U$N)1A.N$N/$N))N//U2P7P	PU
PPUP4Q	A$U6SUT%	)7T 	 T%	%U<UUU
U		Ur6   N)r7   r8   r9   r:   r%   BUILDER_CONFIG_CLASSrK   r^   ru   r   r   r   rA   r6   r#   r   rD   rD   :   s4    %
CBHH  2P'r#   rD   )r
   r   dataclassesr   typingr   pandasr	   r   ru   pyarrow.jsonr   r   r   datasets.configdatasets.tabler   datasets.utils.file_utilsr   utilslogging
get_loggerr7   rG   r   r   r"   BuilderConfigr%   ArrowBasedBuilderrD   r6   r#   r   <module>r      s    	  !       % . 
			*	*8	411/  ''     F'8%% F'r#   