
    h7                        S SK r S SKrS SKJr  SSKJr  SSKJr  \ R                  " S5      R                  r
\" SR                  S	5      5      rS
\S\4S jrS
\S\4S jrS
\S\4S jrS
\S\4S jrS
\S\4S jrS
\S\4S jrS
\S\4S jrS
\S\4S jrS
\S\4S jrS
\S\4S jrS
\S\4S jrS\S\4S jrS\S\4S jrS\S\4S jrS\S\4S jrS\S\4S jrS\S\4S jr S\S\4S jr!S\S\4S jr"S\S\4S  jr#\" 5       4S\S!\\   S\4S" jjr$S%S
\S#\S\4S$ jjr%0 \RL                  \_\RN                  \_\RP                  \_\RR                  \_\RT                  \_\RV                  \_\RX                  \ _\RZ                  \!_\R\                  \"_\R^                  \#_\R`                  \$_\Rb                  \_\Rd                  \_\Rf                  \_\Rh                  \_\Rj                  \_\Rl                  \_\Rn                  \\Rp                  \\Rr                  \\Rt                  \\Rv                  \0Er<g)&    N)Set   )attrs   )	URL_MATCHz0([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)aY  com|org|edu|gov|net|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|pro|tel|travel|xyz|icu|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw|textreturnc                 n    U  H/  n[         R                  " U5      R                  S5      (       a  M/    g   g)NPFT)unicodedatacategory
startswithr	   chars     N/home/james-whalen/.local/lib/python3.13/site-packages/spacy/lang/lex_attrs.pyis_punctr      s0    ##D)44S99      c                 8    U  H  n[        U5      S:  d  M    g   g)N   FT)ordr   s     r   is_asciir       s     t9  r   c                 P   U R                  S5      (       a  U SS  n U R                  SS5      R                  SS5      n U R                  5       (       a  gU R                  S5      S:X  a>  U R	                  S5      u  pUR                  5       (       a  UR                  5       (       a  gg)	N)+-   ±~r   , .T/F)r   replaceisdigitcountsplit)r	   numdenoms      r   like_numr(   '   s    ,--ABx<<R ((b1D||~~zz#!ZZ_
;;==U]]__r   c                     SnX;   $ )N)()[]{}<> )r	   bracketss     r   
is_bracketr4   5   s    7Hr   c                     SnX;   $ )N)"'`   «   »   ‘   ’   ‚   ‛   “   ”   „   ‟   ‹   ›   ❮   ❯''``r2   )r	   quotess     r   is_quoterJ   :   s     IF>r   c                     SnX;   $ )N)r*   r,   r.   r0   r6   r7   r9   r;   r=   r>   r?   rA   rB   rC   rE   rH   r2   )r	   
left_puncts     r   is_left_punctrM   A   s    sJr   c                     SnX;   $ )N)r+   r-   r/   r1   r6   r7   r:   r<   r@   rD   rF   rG   r2   )r	   right_puncts     r   is_right_punctrP   H   s    XKr   c                 N    U  H  n[         R                  " U5      S:w  d  M    g   g)NScFT)r   r   r   s     r   is_currencyrS   M   s'    %-  r   c                 *    [        [        U 5      5      $ N)bool_like_email)r	   s    r   
like_emailrX   U   s    D!""r   c                 
   U R                  S5      (       d  U R                  S5      (       a  gU R                  S5      (       a  [        U 5      S:  a  gU S   S:X  d	  U S   S:X  a  g	S
U ;   a  g	[        [        U 5      5       H  nX   S:X  d  M    O   g	U R                  SS5      S   R	                  SS5      S   nUR                  S5      (       a  gUR                  5       (       a  U[        ;   a  g[        U 5      (       a  gg	)Nzhttp://zhttps://Tzwww.   r   r    F@r   :r!   )	r   lenrangersplitr%   endswithisalpha_tldsr   )r	   itlds      r   like_urlrf   Y   s     y!!T__Z%@%@		 	 SY!^Aw#~bS
d{ 3t97c>  
++c1
a
 
&
&sA
.q
1C
||C
{{}}r   c                 J   [        U 5      S:  a  g/ nSnSnSnU  Hu  nUR                  5       (       a  UR                  5       (       a  SnOSnOUR                  5       (       a  SnOUnX2:X  a  US-  nOSnUnUS	:  d  Md  UR	                  U5        Mw     SR                  U5      $ )
Nd   LONGr   r   Xxdr      )r^   rb   isupperr#   appendjoin)r	   shapelast
shape_charseqr   s         r   
word_shaperu   u   s    
4yCEDJ
C<<>>||~~ 
 
\\^^JJ1HCCD7LL$! " 775>r   stringc                 "    U R                  5       $ rU   lowerrv   s    r   ry   ry      s    <<>r   c                     U S   $ )Nr   r2   rz   s    r   prefixr|      s    !9r   c                     U SS  $ )Nr2   rz   s    r   suffixr      s    "#;r   c                 "    U R                  5       $ rU   )rb   rz   s    r   is_alphar          >>r   c                 "    U R                  5       $ rU   )r#   rz   s    r   is_digitr      r   r   c                 "    U R                  5       $ rU   )islowerrz   s    r   is_lowerr      r   r   c                 "    U R                  5       $ rU   )isspacerz   s    r   is_spacer      r   r   c                 "    U R                  5       $ rU   )istitlerz   s    r   is_titler      r   r   c                 "    U R                  5       $ rU   )rn   rz   s    r   is_upperr      r   r   stopsc                 (    U R                  5       U;   $ rU   rx   )rv   r   s     r   is_stopr      s    <<>U""r   langc                     U$ rU   r2   )r	   r   s     r   get_langr      s	     Kr   )r   )=rer   typingr   r   r   tokenizer_exceptionsr   compilematchrW   setr%   rc   strrV   r   r   r(   r4   rJ   rM   rP   rS   rX   rf   ru   ry   r|   r   r   r   r   r   r   r   r   r   LOWERNORMPREFIXSUFFIXIS_ALPHAIS_DIGITIS_LOWERIS_SPACEIS_TITLEIS_UPPERIS_STOP
LIKE_EMAILLIKE_NUMIS_PUNCTIS_ASCIISHAPE
IS_BRACKETIS_QUOTEIS_LEFT_PUNCTIS_RIGHT_PUNCTIS_CURRENCYLIKE_URL	LEX_ATTRSr2   r   r   <module>r      sQ   	    +jjLMSS? @EuSz	 3 4 3 4 3 4 S T 
3 4     
c d #S #T #3 4 8S S 6# # 3 3 3 3 S T S T S T S T S T S T  ,/5 #C #C #T #3 c 3 	KK	JJ 
LL& 
LL&	
 
NNH 
NNH 
NNH 
NNH 
NNH 
NNH 
MM7 
j 
NNH 
NNH 
NNH  
KK!" 
j#$ 
NNH		.	{	NNH-	r   