
    ^hr                       S r SSKJr  SSKJrJr  SSKrSSKrSSKJ	r	J
r
   " S S\5      r " S S	\5      r " S
 S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\R,                  5      r\ " S S5      5       r " S S5      r\ " S S5      5       rS S S!.       S$S" jjrS%S# jrg)&zTpytokens - A Fast, spec compliant Python 3.12+ tokenizer that runs on older Pythons.    )annotations)	dataclassfieldN)IteratorNewTypec                      \ rS rSrSrg)TokenizeError    N__name__
__module____qualname____firstlineno____static_attributes__r       K/home/james-whalen/.local/lib/python3.13/site-packages/pytokens/__init__.pyr	   r	      s    r   r	   c                      \ rS rSrSrg)IndentationError   r   Nr   r   r   r   r   r      s    sr   r   c                      \ rS rSrSrg)InconsistentUseOfTabsAndSpaces   r   Nr   r   r   r   r   r          r   r   c                      \ rS rSrSrg) DedentDoesNotMatchAnyOuterIndent   r   Nr   r   r   r   r   r      s    #r   r   c                      \ rS rSrSrg)UnterminatedString   r   Nr   r   r   r   r   r      s    r   r   c                      \ rS rSrSrg)UnexpectedEOF   r   Nr   r   r   r   r"   r"      s    Cr   r"   c                      \ rS rSrSrg)!UnexpectedCharacterAfterBackslash   r   Nr   r   r   r   r%   r%      r   r   r%   c                      \ rS rSrSrg)NotAnIndent    r   Nr   r   r   r   r(   r(       s    3r   r(   c                      \ rS rSrSrg)	Underflow#   r   Nr   r   r   r   r+   r+   #   s    r   r+   c                      \ rS rSrSrSrSrSrSrSr	Sr
S	rS
rSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrS S jrS S jrS!S jrSr g)"	TokenType&                           	   
   r
         r         r         r         r          c                     SU R                    3$ )Nz
TokenType.)nameselfs    r   __repr__TokenType.__repr__E   s    DII;''r   c                    U R                   S:X  a  gU R                  5       (       a  gU R                   R                  5       $ )N
identifierNAMEOP)rE   is_operatorupperrF   s    r   to_python_tokenTokenType.to_python_tokenH   s4    99$yy  r   c                d    [         R                  U s=:  =(       a    [         R                  :  $ s  $ N)r.   	_op_start_op_endrF   s    r   rN   TokenType.is_operatorQ   s'    ""T==I,=,=====r   r   NreturnstrrX   bool)!r   r   r   r   
whitespaceindentdedentnewlinenlcommentrT   	semicolonlparenrparenlbracketrbracketlbracerbracecolonoprU   rK   numberstringfstring_startfstring_middlefstring_end	endmarker
errortokenrH   rP   rN   r   r   r   r   r.   r.   &   s    JFFG	
BGIIFFHHFFE	BGJFFMNKIJ(!>r   r.   c                  f    \ rS rSr% S\S'   S\S'   S\S'   S\S'   S\S'   S\S	'   S\S
'   SS jrSrg)TokenU   r.   typeintstart_index	end_index
start_line	start_colend_lineend_colc                   U R                   [        R                  :X  d  U R                   [        R                  :X  a6  U R                  [        U5      :X  a  U R                  [        U5      S-   :X  a  gU R                   [        R                  :X  a9  U R                  [        U5      S-   :X  a  U R                  [        U5      S-   :X  a  gU R                   [        R                  :X  a  gXR                  U R                   $ )Nr0    )	ru   r.   r_   r`   rw   lenrx   r^   rp   )rG   sources     r   to_byte_sliceToken.to_byte_slicea   s     YY)+++tyyILL/H  CK/#f+/1 II)))  CK!O3#f+/1 99	+++&&88r   r   N)r   rY   rX   rY   )r   r   r   r   __annotations__r   r   r   r   r   rs   rs   U   s+    
ONONML9r   rs   c                      \ rS rSr\" S\5      r\" S5      r\" S5      r\" S5      r	\" S5      r
\" S5      r\" S5      rSS	 jrSS
 jrSS jrSS jrSS jrSS jrSS jrSS jrSrg)FStringStatey   Stater0   r1   r2   r3   r4   r5   c                <    [         R                  U l        / U l        g rS   )r   not_fstringstatestackrF   s    r   __init__FStringState.__init__   s    !--
/1
r   c                x    U R                   R                  U R                  5        [        R                  U l        g rS   )r   appendr   r   at_fstring_middlerF   s    r   enter_fstringFStringState.enter_fstring   s%    

$**%!33
r   c                    U R                   [        R                  :X  d   eU R                  R	                  5       U l         g rS   )r   r   at_fstring_endr   poprF   s    r   leave_fstringFStringState.leave_fstring   s,    zz\88888ZZ^^%
r   c                    U R                   [        R                  :X  a%  U R                  R	                  U R                   5        [        R
                  U l         g rS   )r   r   in_fstring_expr_modifierr   r   at_fstring_lbracerF   s    r   !consume_fstring_middle_for_lbrace.FStringState.consume_fstring_middle_for_lbrace   s7    ::>>>JJdjj)!33
r   c                .    [         R                  U l        g rS   )r   r   r   rF   s    r   consume_fstring_middle_for_end+FStringState.consume_fstring_middle_for_end   s    !00
r   c                .    [         R                  U l        g rS   )r   in_fstring_exprr   rF   s    r   consume_lbraceFStringState.consume_lbrace   s    !11
r   c                ^   U R                   [        R                  :X  d   U R                   [        R                  :X  d   e[	        U R
                  5      S:  aA  U R
                  S   [        R                  :X  a   U R
                  R                  5       U l         g [        R                  U l         g )Nr   )r   r   r   r   r   r   r   r   rF   s    r   consume_rbraceFStringState.consume_rbrace   sx    JJ,666zz\BBB	
C
 

Oa

2,"G"GG)DJ%77DJr   c                n    U R                   [        R                  :X  d   e[        R                  U l         g rS   )r   r   r   r   rF   s    r   consume_colonFStringState.consume_colon   s'    zz\99999!::
r   )r   r   NrX   None)r   r   r   r   r   rv   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   y   so    GS!E(KaaAhO$Qx1XN24&4128;r   r   c                  j   \ rS rSr% S\S'   S\S'   SrS\S'   SrS\S	'   S
rS\S'   S
rS\S'   Sr	S\S'   Sr
S\S'   SrS\S'   SrS\S'   \" \S9rS\S'   SrS\S'   \" \S9rS\S'   SrS\S'   \" \S9rS\S'   \" \S9rS\S'   SrS\S '   SrS\S!'   S"rS\S#'   S"rS\S$'   S"rS\S%'   SCS& jrSDS' jrSDS( jrSES) jrSFS* jrSES+ jr SES, jr!S"S-.SGS. jjr"SHS/ jr#SIS0 jr$SES1 jr%SJS2 jr&SJS3 jr'SJS4 jr(SJS5 jr)SJS6 jr*SJS7 jr+SKS8 jr,SLS9 jr-SJS: jr.SJS; jr/SJS< jr0SCS= jr1SCS> jr2SJS? jr3SMS@ jr4SJSA jr5SBr6g)NTokenIterator   rY   r   r[   issue_128233_handlingr   rv   current_index
prev_indexr0   line_numberprev_line_numberbyte_offsetprev_byte_offsetTall_whitespace_on_this_linebracket_level)default_factoryz	list[int]bracket_level_stackNzToken | None
prev_tokenz	list[str]indent_stackdedent_counterr   fstring_statezlist[tuple[str, str]]fstring_prefix_quote_stackz
str | Nonefstring_prefixfstring_quoteFweird_op_caseweird_op_case_nlweird_whitespace_casec                F    U R                   [        U R                  5      :  $ rS   r   r   r   rF   s    r   is_in_boundsTokenIterator.is_in_bounds   s    !!C$444r   c                b    U R                  5       (       d   eU R                  U R                     $ rS   )r   r   r   rF   s    r   peekTokenIterator.peek   s,      """"{{4--..r   c                    U R                   S-   [        U R                  5      :  d   eU R                  U R                   S-      $ Nr0   r   rF   s    r   	peek_nextTokenIterator.peek_next   s>    !!A%DKK(8888{{4--122r   c                X    U =R                   S-  sl         U =R                  S-  sl        g r   r   r   rF   s    r   advanceTokenIterator.advance   s$    aAr   c                X    U =R                   U-  sl         U =R                  U-  sl        g rS   r   )rG   counts     r   
advance_byTokenIterator.advance_by   s$    e#E!r   c                J    U =R                   S-  sl         SU l        SU l        g )Nr0   r   T)r   r   r   rF   s    r   	next_lineTokenIterator.next_line   s#    A+/(r   c                    U R                   U R                     S:X  a&  U =R                  S-  sl        U R                  5         g U R                  5         g )N
r0   )r   r   r   r   rF   s    r   advance_check_newline#TokenIterator.advance_check_newline   s=    ;;t))*d2!#NNLLNr   ignore_casec               2   U H  nU R                   [        U5      -   [        U R                  5      :  a  M4  U R                  U R                   U R                   [        U5      -    nU(       a   UR                  5       nUR                  5       nX4:X  d  M    g   g)NTF)r   r   r   lower)rG   r   optionsoptionsnippets        r   matchTokenIterator.match   s|    F!!CK/#dkk2BBkk$"4"4t7I7ICPVK7WXG!--/   r   c           
     6   U R                   (       aI  UR                  5       (       d4  U[        R                  [        R                  4;  a  [        R
                  OUnU R                   (       a7  U R                  U R                  U R                   nUS:X  a  SU l	        SU l         [        UU R                  U R                  U R                  U R                  U R                  U R                  S9nU[        R                  :X  d  U[        R                   :X  a  U R#                  5         O0U[        R$                  :X  d  U[        R&                  :X  a  OSU l	        X@l        U R                  U l        U R                  U l        U R                  U l        SU l         U$ )Nz F)ru   rw   rx   ry   rz   r{   r|   )r   rN   r.   rk   rl   rj   r   r   r   r   rs   r   r   r   r   r_   r`   r   r\   ra   r   )rG   tok_type
token_type	token_strtokens        r   
make_tokenTokenIterator.make_token   sN    !!((**!1!193C3C DD LL  	  DOOd6H6HIIE!380!&D((,,++%%$$
 y(((H	,DNN---Y=N=N1N/4D,,, $ 0 0 $ 0 0"r   c                    U R                   b@  U R                  c   eU R                  R                  U R                   U R                  45        Xl         X l        g rS   )r   r   r   r   )rG   prefixquotes      r   push_fstring_prefix_quote'TokenIterator.push_fstring_prefix_quote+  sV    *%%111++22$$d&8&89 %"r   c                    U R                   c  U R                  b   e[        e[        U R                  5      S:X  a  SOU R                  R                  5       u  U l         U l        g )Nr   )NN)r   r   r+   r   r   r   rF   s    r   pop_fstring_quoteTokenIterator.pop_fstring_quote5  s`    &%%---O 4223q8 00446 	0T/r   c                   U R                  5       (       a-  U R                  U R                     S:X  a  U R                  5         U R                  5         U R                  (       dI  U R
                  S:  d9  U R                  R                  [        R                  :X  d  U R                  (       a  [        R                  O[        R                  nU R                  U5      nSU l        U$ )Nr   F)r   r   r   r   r   r   r   r   r   r   r   r.   r`   r_   r   )rG   r   r   s      r   r_   TokenIterator.newline@  s    4;;t/A/A#Bd#JLLN %%%%)%%++|/K/KK33 LL "" 	 
+ %r   c                   U R                   S:w  a  [        e[        U R                  5      S:  a9  U R                  R	                  5       nU R                  [        R                  5      $ U R                  [        R                  5      $ )Nr   )	r   r"   r   r   r   r   r.   r^   rp   )rG   _s     r   rp   TokenIterator.endmarkerR  sg    "t  !A%!!%%'A??9#3#344y2233r   c                v   SnU R                   U R                     R                  5       (       a  SnU R                  5         U R	                  5       (       a  U R                   U R                     R                  5       (       d  U R                   U R                     S:X  ar  U R                  5         U R	                  5       (       aM  U R                   U R                     R                  5       (       a  MS  U R                   U R                     S:X  a  Mr  U R	                  5       (       a-  U R                   U R                     S:X  a  U R                  5         U R	                  5       (       Ga  U R                   U R                     R                  5       (       dL  U R                   U R                     S:X  a  U R                   U R                  S-
     R                  5       (       a  U R                  5         U R	                  5       (       a|  U R                   U R                     R                  5       (       a  MS  U R                   U R                     S:X  a1  U R                   U R                  S-
     R                  5       (       a  M  U R                  S-   [        U R                   5      :  GaU  U(       d0  U R                   U R                  S-
     R                  5       (       Ga  U R                   U R                     S:X  d  U R                   U R                     S:X  a  U R                   U R                  S-      R                  5       (       d  U R                  S-   [        U R                   5      :  a  U R                   U R                  S-      S	:X  d   U R                   U R                  S-      S
:X  aO  U R                   U R                  S-      R                  5       (       a   U R                  5         U R                  5         U R	                  5       (       Ga'  U R                   U R                     R                  5       (       dS  U(       d/  U R                   U R                  S-
     R                  5       (       a  U R                   U R                     S:X  a  U R                  5         U R	                  5       (       a  U R                   U R                     R                  5       (       a  MS  U(       d/  U R                   U R                  S-
     R                  5       (       a  U R                   U R                     S:X  a  M  U R	                  5       (       a  U(       d/  U R                   U R                  S-
     R                  5       (       aJ  U R                   U R                     S:X  d  U R                   U R                     S:X  a  U R                  5         U R                  U R                  -
  S:X  a  U R                   U R                  S-
     S:X  a  U R                  S-   [        U R                   5      ::  aJ  U R                   U R                  U R                  S-    S:X  a   U R                  5         U R                  5         U R                  [        R                  5      $ U R                  [        R                  5      $ )NFTr   .r0   eEr1   +-jJz..)r   r   isdigitr   r   r   r   r   r.   rj   rk   )rG   digit_before_decimals     r   decimalTokenIterator.decimal\  s   $;;t))*2244#' LLN !!KK**+3355{{4--.#5LLN	 !!KK**+3355{{4--.#5 4;;t/A/A#Bc#ILLN!!KK**+3355D../36KK 2 2Q 67??AA LLN !!KK**+3355D../36KK 2 2Q 67??AA
 !C$44!T[[1C1Ca1G%H%P%P%R%RD../36;;t112c9 D..23;;==&&*S-==D$6$6$:;sB;;t'9'9A'=>#ED$6$6$:;CCEE LLNLLN !!KK**+3355%T5G5G!5K)L)T)T)V)VKK 2 23s: LLN !!KK**+3355%T5G5G!5K)L)T)T)V)VKK 2 23s: !T[[1C1Ca1G%H%P%P%R%RD../36;;t112c9 LLN 0A5D..23s: ""Q&#dkk*::KK 2 2T5G5G!5KLPTT??9<<00y//00r   c                z   U R                  5         U R                  5         U R                  5       (       a  U R                  U R                     S:X  d:  U R                  U R                     S:X  d  U R                  U R                     S:X  a  U R                  5         U R                  5       (       a]  U R                  U R                     S:X  a  MD  U R                  U R                     S:X  a  Mc  U R                  U R                     S:X  a  M  U R                  5       (       a  U R                  U R                     S:X  d  U R                  U R                     S:X  aR  U R                  5         U R                  5       (       a-  U R                  U R                     S:X  a  U R                  5         U R                  5       (       a  U R                  U R                     S:X  d:  U R                  U R                     S:X  d  U R                  U R                     S:X  a  U R                  5         U R                  5       (       a]  U R                  U R                     S:X  a  MD  U R                  U R                     S:X  a  Mc  U R                  U R                     S:X  a  M  U R	                  [
        R                  5      $ )N01r   r   r   r  r   r   r   r   r   r.   rk   rF   s    r   binaryTokenIterator.binary  s   !!KK**+s2{{4--.#5{{4--.#5LLN !!KK**+s2{{4--.#5{{4--.#5 KK**+s2{{4--.#5LLN  ""t{{43E3E'F#'M!!KK**+s2{{4--.#5{{4--.#5LLN !!KK**+s2{{4--.#5{{4--.#5 y//00r   c                r   U R                  5         U R                  5         U R                  5       (       a  U R                  U R                     S:  a  U R                  U R                     S::  d  U R                  U R                     S:X  a  U R                  5         U R                  5       (       a[  U R                  U R                     S:  a  U R                  U R                     S::  a  Ma  U R                  U R                     S:X  a  M  U R                  5       (       a  U R                  U R                     S:X  d  U R                  U R                     S:X  aR  U R                  5         U R                  5       (       a-  U R                  U R                     S:X  a  U R                  5         U R                  5       (       a  U R                  U R                     S:  a  U R                  U R                     S::  d  U R                  U R                     S:X  a  U R                  5         U R                  5       (       a[  U R                  U R                     S:  a  U R                  U R                     S::  a  Ma  U R                  U R                     S:X  a  M  U R	                  [
        R                  5      $ )Nr
  7r   r   r   r  r  rF   s    r   octalTokenIterator.octal  s   !!KK**+s2D../36{{4--.#5LLN !!KK**+s2D../36{{4--.#5 KK**+s2{{4--.#5LLN  ""t{{43E3E'F#'M!!KK**+s2D../36{{4--.#5LLN !!KK**+s2D../36{{4--.#5 y//00r   c                   U R                  5         U R                  5         U R                  5       (       a  U R                  U R                     [        R
                  ;   d  U R                  U R                     S:X  aq  U R                  5         U R                  5       (       aL  U R                  U R                     [        R
                  ;   a  MR  U R                  U R                     S:X  a  Mq  U R                  5       (       a  U R                  U R                     S:X  d  U R                  U R                     S:X  aR  U R                  5         U R                  5       (       a-  U R                  U R                     S:X  a  U R                  5         U R                  5       (       a  U R                  U R                     [        R
                  ;   d  U R                  U R                     S:X  aq  U R                  5         U R                  5       (       aL  U R                  U R                     [        R
                  ;   a  MR  U R                  U R                     S:X  a  Mq  U R                  [        R                  5      $ )Nr   r   r   r  )	r   r   r   r   rl   	hexdigitsr   r.   rk   rF   s    r   hexadecimalTokenIterator.hexadecimal  s   !!KK**+v/?/??{{4--.#5LLN	 !!KK**+v/?/??{{4--.#5 KK**+s2{{4--.#5LLN  ""t{{43E3E'F#'M!!KK**+v/?/??{{4--.#5LLN	 !!KK**+v/?/??{{4--.#5 y//00r   c                    [        S5       H<  nU R                  U R                  U-      nUS:X  d  US:X  d  M-  U R                  U-   s  $    [        S5      e)Nr2   "'zQuote not found somehow)ranger   r   AssertionError)rG   offsetchars      r   find_opening_quote TokenIterator.find_opening_quote  sU    AhF;;t11F:;Ds{dck))F22 
 677r   c                L   U R                  5       nU R                  U R                  U nU R                  U   nUS-   [        U R                  5      ::  a=  U R                  US-      U:X  a'  U R                  US-      U:X  a  U R                  XS-    OU R                  XS-    nX$4$ )Nr2   r0   r1   )r  r   r   r   )rG   quote_indexr   
quote_charr   s        r   string_prefix_and_quotes&TokenIterator.string_prefix_and_quotes  s    --/T//+>[[-
 a3t{{#33KKa0J>KKa0J>	 KKAo6 [?; 	 }r   c                x   U R                   R                  [        R                  [        R                  4;   a  U R                  5       u  pU R                  X5        [        [        U5      5       H  nU R                  5         M     [        [        U5      5       H  nU R                  5         M     U R                   R                  5         U R                  [        R                  5      $ U R                   R                  [        R                  :X  Ga  U R                  c   e[        U R                  5      S:H  nU R                   nU R#                  5       (       Gas  U R$                  U R                      nUS:X  a  U(       a  [&        eUS:X  a  U R                  5         U R(                  c   eSU R(                  R+                  5       ;  an  U R                   S-   [        U R$                  5      :  aH  U R-                  5       S:X  a4  U R/                  5       S:X  a   U R                  5         U R                  5         U R#                  5       (       a8  U R-                  5       S:X  d$  U R-                  5       S:X  d  U R1                  5         GMC  US:X  a  U R/                  5       S:X  a#  U R                  5         U R                  5         GM  U R                   R3                  5         U R                   U:X  a  U R5                  5       $ U R                  [        R6                  5      $ U R                  c   eU R9                  U R                  5      (       aY  U R                   R;                  5         U R                   U:X  a  U R5                  5       $ U R                  [        R6                  5      $ U R1                  5         U R#                  5       (       a  GMs  [<        eU R                   R                  [        R>                  :X  au  U R                  5         U R@                  RC                  U RD                  5        SU l"        U R                   RG                  5         U R                  [        RH                  5      $ U R                   R                  [        RJ                  :X  a  U R                  c   e[        [        U R                  5      5       H  nU R                  5         M     U RM                  5         U R                   RO                  5         U R                  [        RP                  5      $ U R                   R                  [        RR                  :X  Ga`  U R                   nU R#                  5       (       Ga8  U R$                  U R                      nU R                  c   eUS:X  d  US:X  a  [        U R                  5      S:X  a  US:X  a  U R                   R3                  5         O[        R                  U R                   l        U R                   U:X  a  U R5                  5       $ U R                  [        R6                  5      $ US:X  a>  [        R                  U R                   l        U R                  [        R6                  5      $ U R1                  5         U R#                  5       (       a  GM8  [<        e[U        S	5      e)
Nr0   r   \rN{}r   zUnhandled f-string state)+r   r   r   r   r   r#  r   r  r   r   r   r   r.   rm   r   r   r   r   r   r   r   r   r   r   r   r   fstringrn   r   r   r"   r   r   r   r   r   rg   r   r   r   ro   r   r  )rG   r   r   r   is_single_quoterw   r  s          r   r+  TokenIterator.fstring  s   ##$$(((
 
 !99;MF**693v;' (3u:& ',,.??9#:#:;;##|'E'EE%%111!$"4"45:O,,K##%%{{4#5#564<O,, 4<LLN  ..:::4#6#6#<#<#>> ..2S5EE IIK3. NN,3((**		s*diikS.@224 3;~~'3. **LLN--<#'<<>1#y/G/GHH))555::d0011&&EEG))[8#||~-??9+C+CDD**,i ##%%l  ##|'E'EELLN$$++D,>,>?!"D--/??9#3#344##|'B'BB%%1113t1123 4""$,,.??9#8#899##|'L'LL,,K##%%{{4#5#56))555DLDCKS9K9K5LPQ5Qs{**LLN 4@3O3O**0 ))[8#||~-??9+C+CDDS[/;/K/KD&&,??9+C+CDD**,' ##%%*  788r   c                   U R                  5       u  pU(       a>  U R                  (       a-  U R                  5         U R                  [        R
                  S9$ U H!  nUS:X  d  US:X  d  M  U R                  5       s  $    [        [        U5      5       H  nU R                  5         M     [        [        U5      5       H  nU R                  5         M     [        U5      S:H  nU R                  5       (       a  U R                  U R                     nUS:X  a  U(       a  [        eUS:X  a"  U R                  5         U R                  5         Mi  U R                  U5      (       aJ  [        [        U5      5       H  nU R                  5         M     U R                  [        R                  5      $ U R                  5         U R                  5       (       a  M  [         e)N)r   fFr0   r   r&  )r#  r   r   r   r.   rj   r+  r  r   r   r   r   r   r   r   rl   r"   )rG   r   r   r  r   r,  s         r   rl   TokenIterator.string  sa   557d((LLN??ILL?99Ds{dck||~%  s6{#ALLN $s5z"ALLN # e*/!!;;t112Dt|(( t|**, zz%  s5z*ALLN +y'7'788&&(% !!( r   c                P   U R                   nSnSnU R                  5       (       ag  U R                  U R                      nU R                  5       (       a!  U R	                  5         SnUS:X  d  US:X  a  SnOOU R                  5       (       a  Mg  U R                  5       (       d5  U R                   U:X  a  [
        eU R                  [        R                  5      $ U(       a-  U(       d&  SU l	        U R                  [        R                  5      $ U R                  5       nUS:X  d  US:X  d  U R                  5       (       a  U R                  [        R                  5      $ U R                  XR                    n[        U R                  5      S:X  a  SOU R                  S	   n[        U5      [        U5      :X  a?  [        U5      S:X  a  [
        eXg:w  a  [        eU R                  [        R                  5      $ [        U5      [        U5      :  aT  [        U5      S:  a  Xv;  a  [        eU R                  R                  U5        U R                  [        R                   5      $ [        U R                  5      S:  a  U R                  S	   n[        U5      [        U5      :  a  ["        e[        U5      [        U5      :X  a  OJU R                  R%                  5       n	U =R&                  S
-  sl        [        U R                  5      S:  a  M  U R                  [        R                  5      $ )NFT 	#r&  r   r~   r   r0   )r   r   r   is_whitespacer   r(   r   r.   r\   r   r   
is_newliner   r   r   r   r]   r   r   r   )
rG   rw   saw_whitespacesaw_tab_or_spacer  	next_char
new_indentcurrent_indent
top_indentr   s
             r   r]   TokenIterator.indent  s   (( !!;;t112D!!##!%3;$$,'+$ !!   ""!![0!!??9#7#788 "2)-D&??9#7#788 IIK	yD0DOO4E4E??9#7#788[[/A/AB
"4#4#45:@Q@QRT@Uz?c.11:!#!!+44??9#7#788_s>22>"Q&>+K44$$Z0??9#3#344d''(1,!..r2
z?S_4::z?c*o5%%))+##q(# d''(1, ??9#7#788r   c                    U R                  5       (       a  gU R                  U R                     nUS:H  =(       d)    US:H  =(       d    US:H  =(       d    US:H  =(       d    US:H  $ )NFr3  r   r4  )r7  r   r   )rG   r  s     r   r6  TokenIterator.is_whitespace  sf    ??{{4--.CK t|t| v~ v~	
r   c                   U R                   U R                     S:X  a  gU R                   U R                     S:X  aG  U R                  S-   [        U R                   5      :  a!  U R                   U R                  S-      S:X  a  gg)Nr   Tr   r0   F)r   r   r   rF   s    r   r7  TokenIterator.is_newline  sq    ;;t))*d2KK**+t3""Q&T[[)99D..23t;r   c                   U R                   (       a/  U R                  5         U R                  [        R                  5      $ U R
                  U R                  S  n[        U5       H<  u  p#[        U5      S:  d  M  [        R                  U5      (       a  M2  US:w  d  M:  Un  O   [        U5      nU R                  U5        U R                  [        R                  5      $ )N   r   )r   r   r   r.   rK   r   r   	enumerateordrY   isalnumr   r   )rG   	remainingindexr  lengths        r   rE   TokenIterator.name  s    LLN??9#7#788 KK 2 2 45	$Y/KE4y3s{{4'8'8TS[ 0
 ^Fy3344r   c                    U $ rS   r   rF   s    r   __iter__TokenIterator.__iter__  s    r   c                   U R                   b.  U R                   R                  [        R                  :X  a  [        eU R
                  [        U R                  5      :X  a  U R                   c  U R                  5       $ U R                   R                  [        R                  [        R                  [        R                  1;   a  U R                  5       $ U R                  5       $ U R
                  [        U R                  5      :  a  U R                  5       $ U R                  R                  [        R                  :w  a8  U R                  R                  [        R                  :w  a  U R!                  5       $ U R                  U R
                     nU R"                  (       d  US:X  a  U R%                  5         U R'                  5       (       d  U R                  5       $ U R                  U R
                     nUS:w  aC  SU l        U R                   b/  U R                   R                  [        R*                  :X  a  SU l        US:X  a  U R(                  (       a/  U R%                  5         U R/                  [        R*                  5      $ U R'                  5       (       a  U R1                  5       (       dk  U R"                  (       d  U R                  U R
                     S:X  a  O<U R%                  5         U R'                  5       (       a  U R1                  5       (       d  Mk  U R/                  [        R*                  5      $ U R2                  S:  a4  U =R2                  S-  sl        U R/                  [        R                  5      $ U R1                  5       (       a  U R                  5       $ US:X  GaI  U R%                  5         U R'                  5       (       d  [4        eSnSnU R'                  5       (       a  U R7                  5       (       a  U R%                  5         SnOU(       d  U R1                  5       (       a}  U R                  U R
                     nUS:X  a  U R%                  5         U R%                  5         SnSnU R8                  (       a  U R;                  5         O0U R;                  5         SU l        OOU R'                  5       (       a  M  U(       d  [<        eU R/                  [        R>                  5      $ U R@                  S:X  d  U RB                  (       aU  U RD                  S:X  aE  U R                  R                  [        R                  :X  a  SU l!         U RG                  5       nUb  U$ U R7                  5       (       a  U R'                  5       (       aQ  U R7                  5       (       a<  U R%                  5         U R'                  5       (       a  U R7                  5       (       a  M<  U R/                  [        R>                  5      $ US	;   aS  U R%                  5         U RK                  5       S
:X  a  U R%                  5         U R/                  [        RL                  5      $ US:X  a  U R%                  5         U RK                  5       S:X  a/  U R%                  5         U R/                  [        RL                  5      $ U RK                  5       S:X  a  U R%                  5         U RK                  5       S
:X  a  U R%                  5         U R/                  [        RL                  5      $ US:X  aw  U R%                  5         U RK                  5       S:X  a  U R%                  5         U RK                  5       S
:X  a  U R%                  5         U R/                  [        RL                  5      $ US:X  aw  U R%                  5         U RK                  5       S:X  a  U R%                  5         U RK                  5       S
:X  a  U R%                  5         U R/                  [        RL                  5      $ US:X  aw  U R%                  5         U RK                  5       S:X  a  U R%                  5         U RK                  5       S
:X  a  U R%                  5         U R/                  [        RL                  5      $ US:X  a  U R%                  5         U RK                  5       S:X  a/  U R%                  5         U R/                  [        RL                  5      $ U RK                  5       S
:X  a  U R%                  5         U R/                  [        RL                  5      $ US;   a/  U R%                  5         U R/                  [        RL                  5      $ US:X  a/  U R%                  5         U R/                  [        RL                  5      $ US:X  aD  U R%                  5         U =RD                  S-  sl"        U R/                  [        RN                  5      $ US:X  a[  U R%                  5         U =RD                  S-  sl"        U RD                  S:  a  SU l"        U R/                  [        RP                  5      $ US:X  aD  U R%                  5         U =RD                  S-  sl"        U R/                  [        RR                  5      $ US:X  a[  U R%                  5         U =RD                  S-  sl"        U RD                  S:  a  SU l"        U R/                  [        RT                  5      $ US:X  aD  U R%                  5         U =RD                  S-  sl"        U R/                  [        RV                  5      $ US:X  a  U R%                  5         U RD                  S:X  ab  U R                  R                  [        R                  :X  a:  U R                  RY                  5         U RZ                  R]                  5       U l"        O,U =RD                  S-  sl"        U RD                  S:  a  SU l"        U R/                  [        R^                  5      $ US:X  a  U R%                  5         U RD                  S:X  af  U R                  R                  [        R                  :X  a>  [        R`                  U R                  l        U R/                  [        RL                  5      $ U RK                  5       S
:X  a  U R%                  5         U R/                  [        RL                  5      $ US;   Ga0  U R
                  S-   [        U R                  5      ::  a:  U R                  U R
                  U R
                  S-    S;   a  U Rc                  5       $ U R
                  S-   [        U R                  5      ::  a:  U R                  U R
                  U R
                  S-    S;   a  U Re                  5       $ U R
                  S-   [        U R                  5      ::  a:  U R                  U R
                  U R
                  S-    S;   a  U Rg                  5       $ U Ri                  5       $ U R
                  S-   [        U R                  5      ::  a  U Rk                  SS5      (       d  U R
                  S-   [        U R                  5      ::  a  U Rk                  S S!S"S#S$S%S&S'SS(9	(       dB  U R
                  S)-   [        U R                  5      ::  a,  U Rk                  S*S+S,S-S.S/S0S1SS(9	(       a  U Rm                  5       $ U Ro                  5       $ ! [H         a    S n G	Nf = f)2Nr   r   Tr5  r   r0   r&  F)	r  &|^@%=!~rW  <>/*r  ),;`()[]r)  r*  :z.0123456789r1   )0b0B)0o0O)0x0Xr  r  zb"zb'zr"zr'zf"zf'zu"zu'r   r2   zbr"zbr'zrb"zrb'zfr"zfr'zrf"zrf')8r   ru   r.   rp   StopIterationr   r   r   r_   r`   r^   r   r   r   r   r   r+  r   r   r   r   ra   r   r   r7  r   r"   r6  r   r   r%   r\   r   r   r   r]   r(   r   rj   rc   rd   re   rf   rg   r   r   r   rh   r   r  r  r  r  r   rl   rE   )rG   current_charfound_whitespaceseen_newliner  indent_tokens         r   __next__TokenIterator.__next__  sB   ??&4??+?+?9CVCV+V T[[!11&~~''##!!  ( 
 ~~''||~%DKK 00>>## $$(@(@@""((L,H,HH<<>!{{4#5#56 ))ld.BLLN$$&&||~%;;t'9'9:Lt#%)"OO/,,	0A0AA,0D) 3!!y'8'899##%%doo.?.?22D$6$674? ##%%doo.?.? ??9#4#455 "1$??9#3#344 ??<<>! 4LLN$$&&##  % L##%%%%''LLN'+$%4??+<+<;;t'9'9:Dt|LLN'+$#'L 77((;@8 / ##%%2 $77??9#7#788 "d&@&@""a'""((L,D,DD).D&$#{{} '####%%$*<*<*>*> ##%%$*<*<*>*>??9#7#788HHLLNyy{c!??9<<003LLNyy{c!y||44yy{c!yy{c!??9<<003LLNyy{c!yy{c!??9<<003LLNyy{c!yy{c!??9<<003LLNyy{c!yy{c!??9<<003LLNyy{c!y||44 yy{c!??9<<00:%LLN??9<<00 3LLN??9<<003LLN!#??9#3#3443LLN!#!!A%%&"??9#3#3443LLN!#??9#5#5663LLN!#!!A%%&"??9#5#5663LLN!#??9#3#3443LLN""a'&&,,0L0LL""113%)%=%=%A%A%C"""a'"%%))*D&??9#3#3443LLN""a'&&,,0L0LL+7+P+P""(y||4499;#%LLNy||44=(!!A%T[[)99dkk""T%7%7!%;?? {{}$##a'3t{{+;;""T%7%7!%;AA zz|###a'3t{{+;;""T%7%7!%;AA ''))||~% !#s4;;'77DJJsC<P<P""Q&#dkk*::JJ $  
 ""Q&#dkk*::JJ $  
 ;;= yy{m  $#$s   {0 0| ?| )r   r   r   r   r   r   r   r   r   r   r   r   rZ   rW   r   )r   rv   rX   r   )r   rY   r   r[   rX   r[   )r   r.   rX   rs   )r   rY   r   rY   rX   r   )rX   rs   )rX   rv   )rX   ztuple[str, str])rX   r   )7r   r   r   r   r   r   r   r   r   r   r   r   r   r   listr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r_   rp   r  r  r  r  r  r#  r+  rl   r]   r6  r7  rE   rO  rq  r   r   r   r   r   r      s   KM3JKcKc(,,M3%*4%@@#J##D9L)9NC #("EM<E8=d8S 5S!%NJ% $M:$  M4"d""'4'5/3"0
 8= -^#	
$4Q1f1414108"r9h%N=9~

5$nr   r   T)fstring_tokensr   c               N    [        XS9nU(       a  [        U5      $ [        U5      $ )N)r   )r   itermerge_fstring_tokens)r   rt  r   token_iterators       r   tokenizery  N  s'     #6WNN##//r   c              #    #    U  H  nUR                   [        R                  :w  a  Uv   M'  UnUnSnSnU  HR  nUR                   [        R                  :X  a  US-  nUR                   [        R                  :X  a  US-  nXE:X  d  MP  Un  O   [	        [        R
                  UR                  UR                  UR                  UR                  UR                  UR                  S9v   M     g7f)zETurn post-Python-3.12 FSTRING-* tokens back to a single STRING token.r0   r   )ru   rw   ry   rz   rx   r{   r|   N)ru   r.   rm   ro   rs   rl   rw   ry   rz   rx   r{   r|   )rx  r   start_token	end_tokenfstring_startsfstring_endss         r   rw  rw  [  s     ::000K	#EzzY444!#zzY222!-!	 $ !!#//"--!++))''%%
 	
)  s   B C,A&C,)r   rY   rt  r[   r   r[   rX   Iterator[Token])rx  r   rX   r  )__doc__
__future__r   dataclassesr   r   enumrl   typingr   r   	Exceptionr	   r   r   r   r   r"   r%   r  r(   r+   IntEnumr.   rs   r   r   ry  rw  r   r   r   <module>r     s    Z " (   $ $I # +} * <%5 ; >'7 = - , (M ' < ; '. & % $,> ,>^  9  9  9F2; 2;j \ \ \D  "&	
0
0 
0  	
0
 
0
r   