
    ht                         S SK r S SKJr  S SKJr  S SKrS SKrSSK	J
r
  SSKJr  SSKJr  \(       a  S SKr " S	 S
\\S\4   5      rg)    N)Mapping)TYPE_CHECKING   )config)
map_nested   )TensorFormatterc                      ^  \ rS rSrSU 4S jjrS rS rS rS\4S jr	S\
R                  S	\4S
 jrS\
R                  S	S4S jrS\
R                  S	\4S jrSrU =r$ )TFFormatter    c                 4   > [         TU ]  XS9  X0l        SS Kng )N)featurestoken_per_repo_idr   )super__init__tf_tensor_kwargs
tensorflow)selfr   r   r   tf	__class__s        Z/home/james-whalen/.local/lib/python3.13/site-packages/datasets/formatting/tf_formatter.pyr   TFFormatter.__init__!   s    (P 0    c                   ^^ SS K m[        T[        5      (       aj  T(       ac  [        UU4S jT 5       5      (       a  TR                  " T5      $ [        UU4S jT 5       5      (       a  TR
                  R	                  T5      $ T$ )Nr   c              3      >#    U  Ha  n[        UTR                  5      =(       a?    UR                  TS    R                  :H  =(       a    UR                  TS    R                  :H  v   Mc     g7f)r   N)
isinstanceTensorshapedtype.0xcolumnr   s     r   	<genexpr>+TFFormatter._consolidate.<locals>.<genexpr>*   sU      pvkl
1bii(fQWWq	-GfAGGW]^_W`WfWfLffpvs   A)A,c              3      >#    U  H`  n[        UTR                  TR                  45      =(       a2    UR                  S :H  =(       a    UR                  TS   R                  :H  v   Mb     g7f)r   r   N)r   r   RaggedTensorndimr   r    s     r   r$   r%   .   sZ      A 1ryy"//:;j!jPQPWPW[abc[d[j[jPjjs   A(A+)r   r   listallstackragged)r   r#   r   s    `@r   _consolidateTFFormatter._consolidate&   st    fd## pv   xx''   
 yyv..r   c                    SS K nUc  U$ 0 n[        U[        R                  [        R                  45      (       aC  [        R
                  " UR                  [        R                  5      (       a  SUR                  0nOq[        U[        R                  [        R                  45      (       aB  [        R
                  " UR                  [        R                  5      (       a  SUR                  0n[        R                  (       aR  S[        R                  ;   a>  SS Kn[        XR                   R                   5      (       a  [        R"                  " U5      n[        R$                  (       a,  S[        R                  ;   a  SSKJn  [        X5      (       a  U$ [        R*                  (       a0  S[        R                  ;   a  SSKJnJn  [        XU45      (       a  U$ UR2                  " U40 0 UEU R4                  ED6$ )Nr   r   PILtorchvision)VideoReader
torchcodec)AudioDecoderVideoDecoder)r   r   npnumberndarray
issubdtyper   integerint64floatingfloat32r   PIL_AVAILABLEsysmodules	PIL.ImageImageasarrayTORCHVISION_AVAILABLEtorchvision.ior2   TORCHCODEC_AVAILABLEtorchcodec.decodersr4   r5   convert_to_tensorr   )r   valuer   default_dtyper0   r2   r4   r5   s           r   
_tensorizeTFFormatter._tensorize7   sL   =Lebii455"--UWU_U_:`:`$bhh/M		2::677BMM%++WYWbWb<c<c$bjj1MES[[$8%11

5)''MS[[,H2%--&&<3;;+FF%!=>>##EX-W-WAVAV-WXXr   c                    SS K n[        R                  (       ap  S[        R                  ;   a\  SS Kn[        XR                  5      (       a>  U R                  UR                  5       R                  5       R                  5       S   5      $ [        US5      (       a*  [        XR                  5      (       d  UR                  5       n[        U[        R                  5      (       aE  UR                   ["        :X  a0  U R%                  U Vs/ s H  o@R'                  U5      PM     sn5      $ OK[        U[(        [*        45      (       a0  U R%                  U Vs/ s H  o@R'                  U5      PM     sn5      $ U R                  U5      $ s  snf s  snf )Nr   torch 	__array__)r   r   TORCH_AVAILABLEr?   r@   rN   r   r   rK   detachcpunumpyhasattrrP   r6   r8   r   objectr-   recursive_tensorizer)   tuple)r   data_structr   rN   	substructs        r   _recursive_tensorize TFFormatter._recursive_tensorizeV   s+    !!g&<+||44{'9'9';'?'?'A'G'G'I"'MNN;,,ZYY5W5W%//1Kk2::..  F*((_j)k_jR[*B*B9*M_j)kll +dE]33$$[f%g[fi&>&>y&I[f%ghh{++ *l%gs   FF
rY   c                 ,    [        U R                  USS9$ )NF)map_list)r   r[   )r   rY   s     r   rW   TFFormatter.recursive_tensorizei   s    $33[5QQr   pa_tablereturnc                     U R                  5       R                  U5      nU R                  R                  U5      nU R	                  U5      $ N)numpy_arrow_extractorextract_rowpython_features_decoder
decode_rowrW   )r   r`   rows      r   
format_rowTFFormatter.format_rowl   sB    ((*66x@**55c:'',,r   	tf.Tensorc                     U R                  5       R                  U5      nU R                  R                  X!R                  S   5      nU R                  U5      nU R                  U5      nU$ )Nr   )rd   extract_columnrf   decode_columncolumn_namesrW   r-   )r   r`   r#   s      r   format_columnTFFormatter.format_columnq   sb    ++-<<XF--;;FDYDYZ[D\]))&1""6*r   c                     U R                  5       R                  U5      nU R                  R                  U5      nU R	                  U5      nU H  nU R                  X#   5      X#'   M     U$ rc   )rd   extract_batchrf   decode_batchrW   r-   )r   r`   batchcolumn_names       r   format_batchTFFormatter.format_batchx   sf    **,::8D,,99%@((/ K!%!2!253E!FE !r   )r   )NN)__name__
__module____qualname____firstlineno__r   r-   rK   r[   dictrW   paTabler   ri   rp   rw   __static_attributes____classcell__)r   s   @r   r   r       sp     
"Y>,&Rt R-288 - -
bhh ; RXX '  r   r   rk   )r?   collections.abcr   typingr   rT   r6   pyarrowr~    r   utils.py_utilsr   
formattingr	   r   r   r   rO   r   r   <module>r      sA      #      ' ' ^/';"?@ ^r   