
    h                         % S SK r S SKJr  S SKJrJr  S SKrS SKr	SSK
Jr  SSKJr  SSKJr  SS	KJr  \(       a  S SKrS SKr\" 5       rSq\\   \S
'    " S S\\S\4   5      rg)    N)Mapping)TYPE_CHECKINGOptional   )config)
get_logger)
map_nested   )TensorFormatterDEVICE_MAPPINGc                      ^  \ rS rSrSU 4S jjr\S\\S4   4S j5       rS r	S r
S rS	\4S
 jrS\R                  S\4S jrS\R                  SS4S jrS\R                  S\4S jrSrU =r$ )JaxFormatter&   c                   > [         TU ]  XS9  SS KnSSKJn  [        X&5      (       a  [        SU S[        U5       S35      e[        U[        5      (       a  UO[        UR                  " 5       S   5      U l
        [        c  U R                  5       qU R                  [        [        R                  5       5      ;  a  [        R!                  SU R                   S[        [        R                  5       5       S	[        UR                  " 5       S   5       S
35        [        UR                  " 5       S   5      U l
        X@l        g )N)featurestoken_per_repo_idr   )Devicez	Expected z to be a `str` not z, as `jaxlib.xla_extension.Device` is not serializable neither with `pickle` nor with `dill`. Instead you can surround the device with `str()` to get its string identifier that will be internally mapped to the actual `jaxlib.xla_extension.Device`.zDevice with string identifier z) not listed among the available devices: z), so falling back to the default device: .)super__init__jaxjaxlib.xla_clientr   
isinstance
ValueErrortypestrdevicesdevicer   _map_devices_to_strlistkeysloggerwarningjnp_array_kwargs)selfr   r   r   r$   r   r   	__class__s          [/home/james-whalen/.local/lib/python3.13/site-packages/datasets/formatting/jax_formatter.pyr   JaxFormatter.__init__'   s   (P,f%%F8#6tF|n E? ?  !+63 7 7fSqAQ=R !!557N;;d>#6#6#899NN0 > !4!4!678 9s{{}Q/014
 ckkmA./DK 0    returnzjaxlib.xla_extension.Devicec                  l    SS K n U R                  " 5        Vs0 s H  n[        U5      U_M     sn$ s  snf Nr   )r   r   r   )r   r   s     r'   r    JaxFormatter._map_devices_to_strB   s,    25++-@-FV#-@@@s   1c                    ^^ SS K mSS KJn  [        T[        5      (       a2  T(       a+  [        UU4S jT 5       5      (       a  UR                  TSS9$ T$ )Nr   c              3      >#    U  Ha  n[        UTR                  5      =(       a?    UR                  TS    R                  :H  =(       a    UR                  TS    R                  :H  v   Mc     g7f)r   N)r   Arrayshapedtype).0xcolumnr   s     r'   	<genexpr>,JaxFormatter._consolidate.<locals>.<genexpr>M   sU      pvkl
1cii(fQWWq	-GfAGGW]^_W`WfWfLffpvs   A)A,)axis)r   	jax.numpynumpyr   r    allstack)r%   r5   jnpr   s    ` @r'   _consolidateJaxFormatter._consolidateH   sL    fd## pv   yyay00r)   c                 .   SS K nSS KJn  [        U[        [
        [        S 5      45      (       a  U$ [        U[        R                  [        R                  45      (       aD  [        R                  " UR                  [        R                  5      (       a  UR                  5       $ 0 n[        U[        R                  [        R                  45      (       am  [        R                  " UR                  [        R                  5      (       a9  UR                  R                   (       a  SUR"                  0nOSUR$                  0nOq[        U[        R                  [        R                  45      (       aB  [        R                  " UR                  [        R&                  5      (       a  SUR(                  0n[        R*                  (       aR  S[,        R.                  ;   a>  SS Kn[        XR2                  R2                  5      (       a  [        R4                  " U5      n[        R6                  (       a,  S[,        R.                  ;   a  SSKJn  [        X5      (       a  U$ [        R<                  (       a0  S[,        R.                  ;   a  SSKJ nJ!n  [        XU45      (       a  U$ [D        c  U RG                  5       q"URH                  " [D        U RJ                     5         URL                  " U40 0 UEU RN                  ED6sS S S 5        $ ! , (       d  f       g = f)Nr   r2   PILtorchvision)VideoReader
torchcodec)AudioDecoderVideoDecoder)(r   r9   r:   r   r   bytesr   np	characterndarray
issubdtyper2   tolistnumberintegerr   jax_enable_x64int64int32floatingfloat32PIL_AVAILABLEsysmodules	PIL.ImageImageasarrayTORCHVISION_AVAILABLEtorchvision.iorC   TORCHCODEC_AVAILABLEtorchcodec.decodersrE   rF   r   r   default_devicer   arrayr$   )	r%   valuer   r=   default_dtyperA   rC   rE   rF   s	            r'   
_tensorizeJaxFormatter._tensorizeS   s   ec5$t*566Lbjj9::r}}U[[Z\ZfZf?g?g<<>!ebii455"--UWU_U_:`:` zz((!(#)) 4!(#)) 4		2::677BMM%++WYWbWb<c<c$ckk2MES[[$8%11

5)''MS[[,H2%--&&<3;;+FF%!=>>
 !!557Nt{{ ;< 99UQ&P&P$:O:O&PQ =<<s   !L
Lc                    SS K n[        R                  (       ap  S[        R                  ;   a\  SS Kn[        XR                  5      (       a>  U R                  UR                  5       R                  5       R                  5       S   5      $ [        US5      (       a*  [        XR                  5      (       d  UR                  5       n[        U[        R                   5      (       aE  UR"                  [$        :X  a0  U R'                  U Vs/ s H  o@R)                  U5      PM     sn5      $ OK[        U[*        [,        45      (       a0  U R'                  U Vs/ s H  o@R)                  U5      PM     sn5      $ U R                  U5      $ s  snf s  snf )Nr   torch 	__array__)r   r   TORCH_AVAILABLErU   rV   re   r   Tensorrb   detachcpur:   hasattrr0   rg   rH   rJ   r2   objectr>   recursive_tensorizer    tuple)r%   data_structr   re   	substructs        r'   _recursive_tensorize!JaxFormatter._recursive_tensorize   s+    !!g&<+||44{'9'9';'?'?'A'G'G'I"'MNN;,,ZYY5W5W%//1Kk2::..  F*((_j)k_jR[*B*B9*M_j)kll +dE]33$$[f%g[fi&>&>y&I[f%ghh{++ *l%gs   FF
rp   c                 ,    [        U R                  USS9$ )NF)map_list)r	   rr   )r%   rp   s     r'   rn    JaxFormatter.recursive_tensorize   s    $33[5QQr)   pa_tablec                     U R                  5       R                  U5      nU R                  R                  U5      nU R	                  U5      $ N)numpy_arrow_extractorextract_rowpython_features_decoder
decode_rowrn   )r%   rw   rows      r'   
format_rowJaxFormatter.format_row   sB    ((*66x@**55c:'',,r)   	jax.Arrayc                     U R                  5       R                  U5      nU R                  R                  X!R                  S   5      nU R                  U5      nU R                  U5      nU$ r,   )rz   extract_columnr|   decode_columncolumn_namesrn   r>   )r%   rw   r5   s      r'   format_columnJaxFormatter.format_column   sb    ++-<<XF--;;FDYDYZ[D\]))&1""6*r)   c                     U R                  5       R                  U5      nU R                  R                  U5      nU R	                  U5      nU H  nU R                  X#   5      X#'   M     U$ ry   )rz   extract_batchr|   decode_batchrn   r>   )r%   rw   batchcolumn_names       r'   format_batchJaxFormatter.format_batch   sf    **,::8D,,99%@((/ K!%!2!253E!FE !r)   )r   r$   )NNN)__name__
__module____qualname____firstlineno__r   staticmethoddictr   r   r>   rb   rr   rn   paTabler   r   r   r   __static_attributes____classcell__)r&   s   @r'   r   r   &   s    16 Ac+H&H!I A A
	.R`,&Rt R-288 - -
bhh ; RXX '  r)   r   r   )rU   collections.abcr   typingr   r   r:   rH   pyarrowr    r   utils.loggingr   utils.py_utilsr	   
formattingr   r   jaxlibr"   r   r   __annotations__r   rf   r)   r'   <module>r      s\      # *    & ' ' 	!% %E?7K#@A Er)   