
    hD                        S r SSKrSSKrSSKJr  SSKJrJr  SSKrSSK	J
r
  SSKJr   " S S	\R                  5      rS
 rS rS rS r\R$                  \
R&                  " S5      :  a  S rO\" 5       (       a  S r\" \5      S 5       rS rS rS rS rS rS r\R$                  \
R&                  " S5      :  a  \" \5      S 5       rg\" 5       (       a  \" \5      S 5       rgg)zPExtends `dill` to support pickling more types and produce more consistent dumps.    N)BytesIO)CodeTypeFunctionType)version   )configc                      ^  \ rS rSr\R
                  R                  \R                  R                  R                  5       5      rSr
SS jrU 4S jrS rSrU =r$ )Pickler   Fc                    [        U5      nX0R                  ;  Ga  S[        R                  ;   a'  SS KnX4R
                  L a  [        U5      " [        5        S[        R                  ;   a3  SS Kn[        X5R                  5      (       a  [        U5      " [        5        S[        R                  ;   a'  SS KnX6R                  L a  [        U5      " [        5        S[        R                  ;   a  SS Kn[        X7R                   5      (       a  [        U5      " ["        5        X7R$                  L a  [        U5      " [&        5        [        X7R(                  R*                  5      (       a  [-        USU5      nS[        R                  ;   a3  SS Kn[        X8R0                  5      (       a  [        U5      " [2        5        U[4        L a  [-        USU5      n[6        R8                  R;                  XUS	9  g )
Nregexr   spacytiktokentorch	_orig_modtransformers_torchdynamo_orig_callable)save_persistent_id)typedispatchsysmodulesr   Patternpklregister_save_regexPatternr   
issubclassLanguage_save_spacyLanguager   Encoding_save_tiktokenEncodingr   Tensor_save_torchTensor	Generator_save_torchGeneratornnModulegetattrr   PreTrainedTokenizerBase)_save_transformersPreTrainedTokenizerBaser   dillr
   save)	selfobjr   obj_typer   r   r   r   r   s	            N/home/james-whalen/.local/lib/python3.13/site-packages/datasets/utils/_dill.pyr+   Pickler.save   sW   9==(#++%}},)*<=#++%h77)*=>S[[(000)*@A#++%h55)*;<.)*>? h88!#{C8C,#h(L(LMM)*ST |##;SAC$8JK    c                    >^ U R                   (       a  [        TU ]	  U5      $  [        U5      n[        R                  R                  X5        g ! [         a    SSKJm  [        UU4S jS9n N@f = f)Nr   Hasherc                 ,   > TR                  U S   5      $ )Nr   )hash)xr4   s    r/   <lambda>)Pickler._batch_setitems.<locals>.<lambda>R   s    AaD0Ar1   key)	_legacy_no_dict_keys_sortingsuper_batch_setitemssorted	Exceptiondatasets.fingerprintr4   r*   r
   )r,   itemsr4   	__class__s     @r/   r>   Pickler._batch_setitemsH   sb    ,,7*511	C5ME
 	$$T1	  	C35&ABE	Cs   A A0/A0c                 h    [        U5      [        La   [        R                  R	                  X5        g g N)r   strr*   r
   memoize)r,   r-   s     r/   rH   Pickler.memoizeU   s%    9CLL  +  r1    )T)__name__
__module____qualname____firstlineno__r*   _dillMetaCatchingDictr
   r   copyr<   r+   r>   rH   __static_attributes____classcell__)rC   s   @r/   r
   r
      sF    zz**4<<+@+@+E+E+GHH#( 'LR2, ,r1   r
   c                    ^  U 4S jnU$ )z'Register a custom reducer for the type.c                 .   > U [         R                  T'   U $ rF   )r
   r   )functs    r/   proxypklregister.<locals>.proxy^   s    "r1   rJ   )rW   rX   s   ` r/   r   r   [   s     Lr1   c                  x   [         R                  R                  SS [        R                  " S5      R                  [        R                  " S5      R                  [        R                  " S5      R                  [        R                  " S5      R                  [        R                  " S5      R                  4;   $ )z<Check if the current dill version is in the supported range.N   0.3.6z0.3.7z0.3.8z0.3.9z0.4.0)r   DILL_VERSIONreleaser   parserJ   r1   r/   _is_supported_dill_versionr`   e   s    &&r*g&&g&&g&&g&&g&&/  r1   c                 6    [        USS9R                  U 5        g)zPickle an object to a file.T)recurseN)r
   dumpr-   files     r/   rc   rc   p   s    D$$$S)r1   c                 L    [        5       n[        X5        UR                  5       $ )zPickle an object to a string.)r   rc   getvaluerd   s     r/   dumpsrh   u   s    9DO==?r1   r\   c                 V    [         R                  R                  R                  U5        g rF   )r*   rO   loginfopicklermsgs     r/   rj   rj   ~   s    

C r1   c                 V    [         R                  R                  R                  X5        g rF   )r*   rO   loggertracerl   s     r/   rj   rj      s    

-r1   c                     [        U SU 35         [        U5      4nU R                  [        X!S9  [        U S5        g ! [         a    SSKJn  [        XR
                  S94n NGf = f)NzSe: r   r3   r:   r-   z# Se)rj   r?   r@   rA   r4   r6   save_reduceset)rm   r-   argsr4   s       r/   	_save_setrw      sf    4u/s~ T+  //s,./s   > $A%$A%c                     SS K n[        U SU 35        UR                  UR                  4nU R	                  UR
                  X1S9  [        U S5        g )Nr   zRe: rs   z# Re)r   rj   patternflagsrt   compile)rm   r-   r   rv   s       r/   r   r      sH    4uKK#Dt5r1   c                     SS K n[        U SU 35        UR                  UR                  UR                  UR
                  4nU R                  UR                  X1S9  [        U S5        g )Nr   zEnc: rs   z# Enc)r   rj   name_pat_str_mergeable_ranks_special_tokensrt   r   )rm   r-   r   rv   s       r/   r    r       sZ    5HHcllC$8$8#:M:MND))49r1   c                   ^ SS K mSU4S jjn[        U SU 35        UR                  TR                  :X  aR  UR	                  5       R                  TR                  5      R                  5       R                  5       TR                  4nO-UR	                  5       R                  5       R                  5       4nU R                  X#US9  [        U S5        g )Nr   c                 Z   > TR                  U 5      nU(       a  UR                  U5      nU$ rF   )
from_numpyr   )np_arraydtypetensorr   s      r/   create_torchTensor-_save_torchTensor.<locals>.create_torchTensor   s)    !!(+[['Fr1   zTo: rs   z# TorF   )
r   rj   r   bfloat16detachtofloatcpunumpyrt   )rm   r-   r   rv   r   s       @r/   r"   r"      s     4u
yyENN"

,00288:ENNK

  "((*,*c:r1   c                    ^ SS K mU4S jn[        U SU 35        UR                  5       4nU R                  X#US9  [        U S5        g )Nr   c                 J   > TR                  5       nUR                  U 5        U$ rF   )r#   	set_state)state	generatorr   s     r/   create_torchGenerator3_save_torchGenerator.<locals>.create_torchGenerator   s#    OO%	E"r1   zGe: rs   z# Ge)r   rj   	get_statert   )rm   r-   r   rv   r   s       @r/   r$   r$      sI    
 4uMMOD-=r1   c                    ^ SS K mU4S jn[        U SU 35        UR                  UR                  5       4nU R	                  X#US9  [        U S5        g )Nr   c                    > TR                   R                  U S   S   5      nUR                  U 5      nUR                  U5      $ )Nnlplang)utilget_lang_classfrom_config
from_bytes)r   byteslang_cls	lang_instr   s       r/   create_spacyLanguage1_save_spacyLanguage.<locals>.create_spacyLanguage   sA    ::,,VE]6-BC((0	##E**r1   zSp: rs   z# Sp)r   rj   r   to_bytesrt   )rm   r-   r   rv   r   s       @r/   r   r      sO    +
 4uJJ'D,<r1   c                     [        U SU 35        UR                  nSU;   a  [        US   [        5      (       a  0 US'   U R	                  [        U5      SX!S9  [        U S5        g )NzTok: cacherJ   )r   r-   z# Tok)rj   __dict__
isinstancedictrt   r   )rm   r-   r   s      r/   r)   r)      s_    5LLE%JuW~t<<gS	2U<r1   c                    [         R                  R                  R                  SU 35        UR                  R                  S5      (       d  [        UR                  R                  [        R                  R                  5      5      S:  aJ  UR                  R                  [        R                  R                  5      S   R                  S5      (       d  UR                  S:X  a  SO([        R                  R                  UR                  5      nSn[         R                  R                  (       Gae  [        US5      (       a  UR                  UR                   UR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  UR0                  UUR                  U[2        R4                  S	:  a  UR6                  OUR8                  UR:                  UR<                  4nGOUR                  UR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  UR0                  UUR                  UUR8                  UR:                  UR<                  4nOUR                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  UR0                  UUR                  UUR8                  UR:                  UR<                  4nU R?                  [@        XAS
9  [         R                  R                  R                  S5        g)z
From dill._dill.save_code
This is a modified version that removes the origin (filename + line no.)
of functions created in notebooks or shells for example.
zCo: <   
ipykernel_<lambda> co_posonlyargcount)r[   
   rs   # CoN)!r*   rO   rj   rk   co_filename
startswithlensplitospathsepco_namebasenamePY3hasattrco_argcountr   co_kwonlyargcount
co_nlocalsco_stacksizeco_flagsco_code	co_constsco_namesco_varnamesr   version_infoco_linetable	co_lnotabco_freevarsco_cellvarsrt   r   rm   r-   r   co_firstlinenorv   s        r/   
_save_coder      sc    	

d3%L)& ))#..COO))"''++67!;OO))"''++6r:EElSS{{j(  !!#//2 	 ::>>>s011OO**))NN$$LLKKMMLLOOKK"(+(8(8G(CC$$OOOO!( OO))NN$$LLKKMMLLOOKK"MMOOOO&   D  	Hd4

F#r1   c                 >
   [         R                  R                  R                  U SU5        UR                  R                  S5      (       d  [        UR                  R                  [        R                  R                  5      5      S:  aJ  UR                  R                  [        R                  R                  5      S   R                  S5      (       d  UR                  S:X  a  SO([        R                  R                  UR                  5      nSn[        US5      (       a  UR                  UR                  UR                   UR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  UR0                  UUR                  UR2                  UUR                  UR4                  UR6                  UR8                  UR:                  UR<                  4nGO[        US	5      (       a  UR                  UR                  UR                   UR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  UR0                  UUR                  UR2                  UUR                  UR8                  UR:                  UR<                  4nGO[        US
5      (       a  UR                  UR                  UR                   UR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  UR0                  UUR                  UUR                  UR:                  UR<                  4nGOC[        US5      (       a  UR                  UR                   UR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  UR0                  UUR                  UUR>                  UR:                  UR<                  4nOUR                  UR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  UR0                  UUR                  UUR>                  UR:                  UR<                  4nU RA                  [         R                  RB                  XAS9  [         R                  R                  R                  U S5        g )NzCo: %sr   r   r   r   r   r   co_endlinetableco_exceptiontabler   r   rs   r   )"r*   rO   rp   rq   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   co_qualnamer   co_columntabler   r   r   r   rt   _create_coder   s        r/   	save_coder   D  s   

37, ))#..COO))"''++67!;OO))"''++6r:EElSS{{j(  !!#//2 	  3)**  &&%%    ##""%%+D. S-..  &&%%    %%'D* S.))  &&%%    #D& S.//&&%%  !D( %%  D$ 	DJJ33TC

0r1   )__doc__r   r   ior   typesr   r   r*   	packagingr   r   r   r
   r   r`   rc   rh   r]   r_   rj   ru   rw   r   r    r"   r$   r   r)   r   r   rJ   r1   r/   <module>r      s   W 	 
  (   =,dll =,@*
 
w//!  !!. S & 
w//\ \|  !!P P "r1   