
    h                         S SK r S SKJr  S SKJr  S SKrS SKrS SKJ	r	  \R                  R                  R                  \5      r\ " S S\R                  5      5       r " S S\R"                  5      rg)	    N)	dataclass)Optional)
table_castc                   V   ^  \ rS rSr% SrSr\\R                     \	S'   U 4S jr
SrU =r$ )ArrowConfig   zBuilderConfig for Arrow.Nfeaturesc                 "   > [         TU ]  5         g N)super__post_init__)self	__class__s    _/home/james-whalen/.local/lib/python3.13/site-packages/datasets/packaged_modules/arrow/arrow.pyr   ArrowConfig.__post_init__   s         )__name__
__module____qualname____firstlineno____doc__r	   r   datasetsFeatures__annotations__r   __static_attributes____classcell__)r   s   @r   r   r      s%    ",0Hhx(()0   r   r   c                   d    \ rS rSr\rS rS rS\R                  S\R                  4S jr
S rSrg	)
Arrow   c                 R    [         R                  " U R                  R                  S9$ )N)r	   )r   DatasetInfoconfigr	   )r   s    r   _infoArrow._info   s    ##T[[-A-ABBr   c                    U R                   R                  (       d"  [        SU R                   R                   35      eSUR                  l        UR                  U R                   R                  5      n/ nUR                  5        GH  u  pE[        U[        5      (       a  U/nU Vs/ s H  oaR                  U5      PM     nnU R                  R                  c  [        R                  R                  U5       Hp  n[        US5       n [         R"                  R%                  U5      nSSS5        [.        R0                  R3                  WR4                  5      U R                  l          O   UR7                  [.        R8                  " USU0S95        GM     U$ s  snf ! [&        [         R(                  R*                  4 a"    [         R"                  R-                  U5      n Nf = f! , (       d  f       N= f)z-We handle string, list and dicts in datafilesz=At least one data file must be specified, but got data_files=TNrbfiles)name
gen_kwargs)r#   
data_files
ValueErrordownload_configextract_on_the_flydownload_and_extractitems
isinstancestr
iter_filesinfor	   	itertoolschainfrom_iterableopenpaipcopen_streamOSErrorlibArrowInvalid	open_filer   r   from_arrow_schemaschemaappendSplitGenerator)	r   
dl_managerr+   splits
split_namer(   filefreaders	            r   _split_generatorsArrow._split_generators   s   {{%%\]a]h]h]s]s\tuvv8<
""544T[[5K5KL
!+!1!1!3J%%%=BCUT**40UECyy!!)%OO99%@DdD)Q9%'VV%7%7%:F *
 *2):):)L)LV]])[DII& A MM(11zwX]N^_` "4   D !()<)<= 9%'VV%5%5a%8F9 *)s1   (F	G)F  AG&	#G)%G&	&G))
G7pa_tablereturnc                     U R                   R                  b)  [        XR                   R                  R                  5      nU$ r   )r4   r	   r   arrow_schema)r   rL   s     r   _cast_tableArrow._cast_table7   s3    99) "(II,>,>,K,KLHr   c              #     ^
#    [        [        R                  R                  U5      5       H  u  p#[	        US5       n  [
        R                  R                  U5      n[        U5       H?  u  pg[
        R                  R                  U/5      nU SU 3U R!                  U5      4v   MA      S S S 5        M     g ! [        [
        R                  R                  4 aA    [
        R                  R                  U5      m
U
4S j[        T
R                  5       5       n Nf = f! ["         a-  n	[$        R'                  SU S[)        U	5       SU	 35        e S n	A	ff = f! , (       d  f       GMA  = f7f)Nr'   c              3   F   >#    U  H  nTR                  U5      v   M     g 7fr   )	get_batch).0irI   s     r   	<genexpr>)Arrow._generate_tables.<locals>.<genexpr>F   s!     "a@`16#3#3A#6#6@`s   !_zFailed to read file 'z' with error z: )	enumerater5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   rangenum_record_batchesTablefrom_batchesrP   r,   loggererrortype)r   r(   file_idxrG   rH   batches	batch_idxrecord_batchrL   erI   s             @r   _generate_tablesArrow._generate_tables>   s7    '	(E(Ee(LMNHdD!Qb"$&&"4"4Q"7 4=W3E/	#%88#8#8,#H "*
!I;79I9I(9SSS 4F "! N
 $RVV%8%89 b!#!1!1!!4"afF_F_@`"ab " LL#8mDQRG9TVWXVY!Z[ "!sZ   :E0E B;AD#.E0;A"D D#D  D##
E-(EEE
E-	'	E0r   N)r   r   r   r   r   BUILDER_CONFIG_CLASSr$   rJ   r9   r]   rP   rg   r   r   r   r   r   r      s3    &C2BHH  r   r   )r5   dataclassesr   typingr   pyarrowr9   r   datasets.tabler   utilslogging
get_loggerr   r_   BuilderConfigr   ArrowBasedBuilderr   r   r   r   <module>rs      sf     !    % 
			*	*8	4  (((    7H&& 7r   