
    hJ                        S SK r S SKJr  S SKJrJr  S SKJr  S rS r	S r
S rS	 rS
 rS rS rS rS rS rS rS rS rS rS r\ R0                  R3                  SSS0SS0SS0S//5      S 5       rS rS rS rS rg)     N)LEMMA)DocToken)Vocabc                    SnSSSSS.nU " U5      n[        U5      S:X  d   eUR                  5        nUR                  USS	 US
9  UR                  US	S US
9  S S S 5        [        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R
                  S:X  d   eUS   R                  S:X  d   e[        US   R                  5      S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R
                  S:X  d   e[        US   R                  5      S:X  d   eUS   R                  S:X  d   eg ! , (       d  f       GN= f)N-WKRO played songs by the beach boys all nightNAMEDr   TYPEzNumber=Plur)taglemmaent_typemorph	         attrs   the beach boyszthe beach boys    z	all night)	len
retokenizemergetexttext_with_wstag_lemma_strr   )en_tokenizerr   r   docretokenizers        _/home/james-whalen/.local/lib/python3.13/site-packages/spacy/tests/doc/test_retokenize_merge.pytest_doc_retokenize_merger#      s   :D	E t
Cs8q==		[#a(%0#a(%0 
 s8q==q6;;****q6"3333q6;;'!!!q6==G###s1v||---q6;;+%%%q6+---q6;;'!!!s1v||---q6==G### 
	s   'E
E'c                    SnSSSS.nU " U5      n[        U5      S:X  d   eUR                  5        nUR                  USS US	9  S
S
S
5        U H  nUR                  UR                  R                  :  a'  U[        UR                  R                  5      ;   d   eMN  UR                  UR                  R                  :  d  Mt  U[        UR                  R                  5      ;   a  M   e   g
! , (       d  f       N= f)z3Test that attachments work correctly after merging.r   r	   r   r
   r   r   r   r   r   r   r   N)r   r   r   iheadlistleftsrights)r   r   r   r    r!   words         r"   "test_doc_retokenize_merge_childrenr,   "   s    :Dg6BE
t
Cs8q==		[#a(%0 
66DIIKK4		0000VVdiikk!4		 0 01111	  
	s   C..
C<c                     SnU " U5      nUR                  5        nUR                  USS SSS.S9  UR                  USS	 SSS.S9  S S S 5        g ! , (       d  f       g = f)
Nz through North and South Carolina   r    ORGr   r   r         )r   r   r   r   r    r!   s       r"   test_doc_retokenize_merge_hangr5   1   se    -D
t
C		[#a(BE*JK#a(BE*JK 
		s   -A
Ac                     U " S5      nUR                  5        nUR                  USS 5        S S S 5        [        U5      S:X  d   eUS   R                  S:X  d   eg ! , (       d  f       N5= f)Nr   r   r   r   )r   r   r   r   )r   r    r!   s      r"   test_doc_retokenize_retokenizerr7   9   sb    
F
GC		[#a(# 
s8q==q6;;**** 
	s   A
A+c                 p   U " S5      n[         SSUR                  R                  S   0nUR                  5        nUR	                  USS US9  S S S 5        [        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eg ! , (       d  f       N_= f)	Nr   boysENT_TYPEr0   r   r   r   r   )	r   vocabstringsr   r   r   r   r   	ent_type_)r   r    r   r!   s       r"   %test_doc_retokenize_retokenizer_attrsr>   A   s    
F
GCFJ		(9(9%(@AE		[#a(%0 
s8q==q6;;****q6==F"""q6u$$$ 
	s   B''
B5c                    U " S5      n[        S U 5       5      (       a   eUR                  5        nUR                  USS SSS.S9  S	S	S	5        US   R                  S
:X  d   eUS   R                  S:X  d   eUS   R
                  (       d   e[        UR                  S
/S9nUS   R
                  (       d   eg	! , (       d  f       Nz= f)z8Test that lexical attributes can be changed (see #2390).zWKRO played beach boys songsc              3   8   #    U  H  oR                   v   M     g 7fNis_stop).0tokens     r"   	<genexpr>0test_doc_retokenize_lex_attrs.<locals>.<genexpr>P   s     2cU==c   r3   r   r9   T)r   IS_STOPr   Nz
beach boyswordsr   )anyr   r   r   r   rC   r   r;   )r   r    r!   new_docs       r"   test_doc_retokenize_lex_attrsrN   M   s    
5
6C2c22222		[#a(Ft*LM 
q6;;,&&&q6==F"""q6>>>#))L>2G1: 
	s   B==
Cc                 ~   Sn/ SQnS/[        U5      -  nU " U5      n[        UR                  U Vs/ s H  oUR                  PM     snX#S9n[        U5      S:X  d   eUS   R                  R                  S:X  d   eUS   R                  R                  S	:X  d   eUR                  5        nS
SSS.nUR                  USS US9  S S S 5        [        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  R                  S	:X  d   eUS   R                  S:X  d   eg s  snf ! , (       d  f       Nn= f)NLos Angeles start.r2   r3   r3   r3   deprK   headsdepsr   r   Angelesr2   startNNPLos AngelesGPEr%   r3   r   r.   )r   r   r;   r   r'   r   r   r=   )	r   r   rT   rU   tokenstr    r!   r   s	            r"   &test_doc_retokenize_spans_merge_tokensr]   Z   s1   DE7SZD$F
fll6":6a666":%
SCs8q==q6;;y(((q6;;w&&&		[5I#a(%0 
 s8q==q6;;-'''q6;;w&&&q6u$$$ #; 
	s   D)
,D..
D<c           
         / SQnU Vs/ s H  o"R                  5       PM     nn/ SQnS/[        U5      -  n/ SQn/ SQn[        XXgXEUS9n[        U5      S:X  d   eUS   R                  S	:X  d   eUS   R                  S
:X  d   eUS   R
                  S:X  d   eUS   R                  S:X  d   eUR                  5        n	U	R                  USS 5        S S S 5        [        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R
                  S:X  d   eUS   R                  S:X  d   e[        XXgXEUS9n[        U5      S:X  d   eUS   R                  S	:X  d   eUS   R                  S
:X  d   eUS   R
                  S:X  d   eUS   R                  S:X  d   eUR                  5        n	U	R                  USS 5        U	R                  USS 5        S S S 5        [        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R
                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R
                  S:X  d   eUS   R                  S:X  d   eg s  snf ! , (       d  f       GN= f! , (       d  f       N= f)N)TheplayersrW   .rQ   rR   )DTNNVBZra   )DETNOUNVERBPUNCT)rK   tagsposrT   rU   lemmasr   r   r_   rb   re   ther3   r.   zThe playersrc   rf   zthe playersr2   zstart .rd   rg   )	lowerr   r   r   r   pos_r   r   r   )
en_vocabrK   r\   rk   rT   rU   ri   rj   r    r!   s
             r"   4test_doc_retokenize_spans_merge_tokens_default_attrsrp   l   s   ,E!&'AggiF'E7SZD#D
*C
DRXC s8q==q6;;%q6;;$q6;;%q6==E!!!		[#a(# 
s8q==q6;;-'''q6;;$q6;;&   q6==M)))
DRXC s8q==q6;;%q6;;$q6;;%q6==E!!!		[#a(##a(# 
 s8q==q6;;-'''q6;;$q6;;&   q6==M)))q6;;)###q6;;%q6;;&   q6==I%%%O ( 
	 
	s   J. J3)K3
K
Kc                    / SQn/ SQnS/[        U5      -  n[        XX#S9n[        U5      S:X  d   eUR                  5        nUS   R                  SSS	.nUR	                  US
S US9  S S S 5        [        U5      S:X  d   eUS   R
                  R                  S:X  d   eUS   R
                  R                  S:X  d   eUS   R
                  R                  S
:X  d   eUS
   R
                  R                  S:X  d   eUS   R
                  R                  S;   d   eUS   R
                  R                  S:X  d   eg ! , (       d  f       N= f)N)Ifoundapilatesclassnearworkra   )r2   r2   r   r   r2   r   r   r2   rR   rS      r   zpilates classOr%   r.   r   r   r   r   r2   r3   )r2   r.   )r   r   r   r   r   r'   r&   )ro   rK   rT   rU   r    r!   r   s          r"   %test_doc_retokenize_spans_merge_headsr{      s-   HE$E7SZD
h5
<Cs8q==		[Ao3O#a(%0 
 s8q==q6;;==Aq6;;==Aq6;;==Aq6;;==Aq6;;==F"""q6;;==A 
	s   'D>>
Ec           	      0   SnU " U5      n[         R                  " [        5         UR                  5        nUR	                  USS SSSS.S9  UR	                  USS	 SSSS.S9  S S S 5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)
NrP   r   r3   rX   rY   rZ   r%   r   r2   )pytestraises
ValueErrorr   r   r4   s       r"   ,test_doc_retokenize_spans_merge_non_disjointr      s    D
t
C	z	"^^Aa#mO   Aa#mO    
#	" 
#	"s"   B/A6%B6
B	 B
Bc                 j   Sn/ SQnS/[        U5      -  nU " U5      n[        UR                  U Vs/ s H  oUR                  PM     snX#S9nUS   R                  R
                  S:X  d   eUR                  5        nSSS	S
.nUR                  USS US9  S S S 5        US   R                  R
                  S:X  d   eSn/ SQnS/[        U5      -  nU " U5      n[        UR                  U Vs/ s H  oUR                  PM     snX#S9nUR                  5        nUR                   H6  n	U	R                  U	R                  U	R                  S
.nUR                  XS9  M8     S S S 5        Sn/ SQnS/[        U5      -  nU " U5      n[        UR                  U Vs/ s H  oUR                  PM     snX#S9nUR                  5        nUR                   H  n	UR                  U	5        M     S S S 5        g s  snf ! , (       d  f       GNh= fs  snf ! , (       d  f       N= fs  snf ! , (       d  f       g = f)N.displaCy is a parse tool built with Javascript)r2   r2   r   r   r2   r   r   r   rR   rS   r   r2   NPtoolrz   r%   r3   r   r   zmdisplaCy is a lightweight and modern dependency parse tree visualization tool built with CSS3 and JavaScript.)r2   r2   
   r   r.   r.   r   r   r   r   r2   r            r   r2   zFOne test with entities like New York City so the ents list is not void)r2   r2   r2   r3   r.   r   r   r   r   r   r   r   r2   r   r   )r   r   r;   r   r'   r&   r   r   entslabel_r   )
r   r   rT   rU   r[   r\   r    r!   r   ents
             r"   "test_doc_retokenize_span_np_mergesr      s   ;D$E7SZD$F
fll6":6a666":%
SCq6;;==A		[v3?#a(%0 
 q6;;==AzDGE7SZD$F
fll6":6a666":%
SC		[88CJJTEc/  

 TD?E7SZD$F
fll6":6a666":%
SC		[88Cc"  
	- #;		 #;		 #;		s6   G2
<G7.H	
AHH
%H$7
H
H$
H2c           	      >   Sn/ SQnS/[        U5      -  n/ SQn/ SQnS/[        U5      -  nSUS'   S	US
'   SUS'   SUS'   S	US'   U " U5      n[        UR                  U Vs/ s H  owR                  PM     snUUUUS9n[        U5      S:X  d   eUR	                  5        n	UR
                   H=  n
[        S U
 5       5      nU
R                  R                  US.nU	R                  XS9  M?     S S S 5        [        U5      S:X  d   eg s  snf ! , (       d  f       N%= f)NzPStewart Lee is a stand up comedian who lives in England and loves Joe Pasquale.
)r2   r3   r3   r   r   r   r3   ry   r   ry   r   ry   ry      r   r3      rR   )rX   rX   rd   rb   VBRPrc   WPrd   INrX   CCrd   rX   rX   ra   SP))PERSONr   r3   )rZ   r   r   )r   r   r   rz   zB-PERSONr   zI-PERSONr2   zB-GPEr   r   r   )rK   rT   rU   ri   r      c              3   8   #    U  H  oR                   v   M     g 7frA   )r=   )rD   ws     r"   rF   9test_doc_retokenize_spans_entity_merge.<locals>.<genexpr>   s     41;;rH   r1   r   r   )
r   r   r;   r   r   r   maxrootr   r   )r   r   rT   rU   ri   r   r[   r\   r    r!   r   r   r   s                r"   &test_doc_retokenize_spans_entity_merger      s"   ^DBE7SZDxDBD53u:DDGDGDHDHDH$F
%&v!vvv&C s8r>>		[88C444H!hhoo8DEc/  
 s8r>> ' 
	s   "D	
!AD
Dc                 @   / SQn[        [        5       US9nUR                  R                  R	                  S5      SS4UR                  R                  R	                  S5      SS4/Ul        US   R                  S:X  d   eUS	   R                  S
:X  d   eUS   R                  S
:X  d   eUS   R                  S:X  d   eUR                  5        nUR                  USS 5        S S S 5        [        U5      [        U5      S	-
  :X  d   eUS   R                  S:X  d   eUS	   R                  S
:X  d   e/ SQn[        [        5       US9nUR                  5        nSS	S.nUR                  USS US9  UR                  USS US9  S S S 5        US   R                  S:X  d   eUS	   R                  S
:X  d   e/ SQn[        [        5       US9nUR                  R                  R	                  S5      SS4UR                  R                  R	                  S5      SS4/Ul        US   R                  S:X  d   eUS   R                  S
:X  d   eUS   R                  S:X  d   eUS   R                  S
:X  d   eUR                  5        nUR                  USS 5        UR                  USS 5        UR                  USS 5        S S S 5        [        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   e/ SQn/ SQnS/[        U5      -  nSUS'   SUS'   SUS'   SUS'   S/[        U5      -  nU R                  R	                  S5        U R                  R	                  S5        U R                  R	                  S5        [        XXWUS9nUSS R                  US   :X  d   eUSS R                  US   :X  d   eUR                  5        nUR                  USS 5        UR                  USS 5        UR                  USS 5        S S S 5        [        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S
:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   e/ SQn/ SQnS/[        U5      -  nSUS'   SUS'   SUS'   SUS'   S/[        U5      -  n[        XXWUS9nUR                  5        nUR                  USS 5        UR                  USS 5        S S S 5        [        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eg ! , (       d  f       GN= f! , (       d  f       GNZ= f! , (       d  f       GN.= f! , (       d  f       GN= f! , (       d  f       N= f)N)rt   bcderJ   zent-abcr   r.   zent-dr   Br2   rr   r3   )r   ent_iobr   r   )	rt   r   r   r   r   fghr&   zent-dezent-fgr   r   r   )	r   r   r.   r   r   r   r   r   r   rz   zB-ent-dezI-ent-dezB-ent-fgzI-ent-fgrR   )rK   rT   rU   r   )	r   r   r.   r   r   r   r   r   r   )r   r   r;   r<   addr   ent_iob_r   r   r   r=   r   )ro   rK   r    r!   r   rT   r   rU   s           r"   *test_doc_retokenize_spans_entity_merge_iobr      s   %E
egU
#C					y	)1a0					w	'A.CH q6??c!!!q6??c!!!q6??c!!!q6??c!!!		[#a(# 
s8s5zA~%%%q6??c!!!q6??c!!! &E
egU
#C		[&15#a(%0#a(%0 
 q6??c!!!q6??c!!! :E
egU
#C					x	(!Q/					x	(!Q/CH q6??c!!!q6??c!!!q6??c!!!q6??c!!!		[#a(##a(##a(# 
 s8q==q6??c!!!q6x'''q6??c!!!q6x''' :E'E53u:DDGDGDGDG7SZD""
h5$
GCq8==CF"""q8==CF"""		[#a(##a(##a(# 
 s8q==q6??c!!!q6x'''q6??c!!!q6x'''q6??c!!!q6x''' :E'E53u:DDGDGDGDG7SZD
h5$
GC		[#a(##a(# 
 s8q==q6??c!!!q6x'''q6??c!!!q6x'''e 
	 
	& 
	2 
	, 
	s<   W,W=W+=W=0)X
W
W(+
W:=
X
Xc                    Sn/ SQn/ SQnU " U5      n[        UR                  U Vs/ s H  oUR                  PM     snX#S9n[        UR                  5      u  px[        U5      n	[        U5      n
UR                  5        nSSS.nUR                  USS US	9  UR                  US
S  US	9  S S S 5        [        UR                  5      u  px[        U5      U	S-
  :X  d   e[        U5      U
S-
  :X  d   eg s  snf ! , (       d  f       NS= f)NzOStewart Lee is a stand up comedian. He lives in England and loves Joe Pasquale.)r2   r3   r3   r   r3   r   r   r3   r   r   r   r   r   r   r   r   r   )compoundnsubjROOTdetamodprtattrpunctr   r   preppobjccconjr   dobjr   rS   noner1   r   r3   r   r2   )r   r;   r   r(   sentsr   r   r   )r   r   rT   rU   r[   r\   r    sent1sent2init_len	init_len2r!   r   s                r"   5test_doc_retokenize_spans_sentence_update_after_merger   [  s    \DBE)D $F
fll6":6a666":%
SC		?LE5zHE
I		[ f5#a(%0#bc(%0 
 		?LEu:A%%%u:Q&&& #; 
	s   C3
?,C88
Dc                    Sn/ SQn/ SQnU " U5      n[        UR                  U Vs/ s H  oUR                  PM     snX#S9n[        UR                  5      S   n[        [        UR                  R                  5      5      nUR                  5        n	SSS.n
U	R                  USS U
S	9  S S S 5        [        [        UR                  R                  5      5      US
-
  :X  d   eg s  snf ! , (       d  f       NE= f)NzNStewart Lee is a stand up comedian who lives in England and loves Joe Pasquale)r2   r3   r3   r   r   r   r3   ry   r   ry   r   ry   ry   r   r   )r   r   r   r   r   r   r   r   relclr   r   r   r   r   r   rS   r   r   r1   r3   r   r2   )
r   r;   r   r(   r   r   r   subtreer   r   )r   r   rT   rU   r[   r\   r    r   r   r!   r   s              r"   ,test_doc_retokenize_spans_subtree_size_checkr   q  s    [D;ED $F
fll6":6a666":%
SCOAE4

**+,H		[ f5#a(%0 
 tEJJ&&'(HqL888 #; 
	s   C%
C**
C8c                 `   [         R                  " SSSS9  [         R                  " SSSS9  [        U / SQS9nUR                  5        nS	SS
S.S.nUR	                  USS US9  S S S 5        US   R
                  S	:X  d   eUS   R                  R                  SL d   eUS   R                  R                  S
:X  d   e[        U / SQS9nUR                  5        nUR	                  USS SSS
S.0S9  UR	                  USS SS SS.0S9  S S S 5        US   R                  R                  SL d   eUS   R                  R                  S
:X  d   eUS   R                  R                  b   eUS   R                  R                  S:X  d   eg ! , (       d  f       GN/= f! , (       d  f       N= f)Nrt   FT)defaultforcer   nothinghelloworld!rJ   hello world1)rt   r   )r   _r   r3   r   )r   r   r   r   r   r   2r2   )	r   set_extensionr   r   r   r   r   rt   r   )ro   r    r!   r   s       r"   )test_doc_retokenize_merge_extension_attrsr     s   	U$7	Yd;
h5
6C		['Ds.CD#a(%0 
 q6==M)))q688::q688::
h:
;C		[#a(3d0E*FG#a(3d0E*FG 
 q688::q688::q688::q688:: 
	 
	s   	F1F
F
F-underscore_attrsrt   xr   r   r2   c                 b   [         R                  " SS SS9  [         R                  " SS SS9  [        U / SQS	9nS
U0n[        R                  " [
        5         UR                  5        nUR                  USS US9  S S S 5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)Nrt   c                     U $ rA    r   s    r"   <lambda>Ctest_doc_retokenize_merge_extension_attrs_invalid.<locals>.<lambda>      a    T)getterr   r   c                     U $ rA   r   r   s    r"   r   r     r   r   )methodr   r   rJ   r   r   r3   r   )r   r   r   r}   r~   r   r   r   )ro   r   r    r   r!   s        r"   1test_doc_retokenize_merge_extension_attrs_invalidr     s    	Kt<	Kt<
h5
6C"#E	z	"^^c!Ahe4  
#	" 
#	"s$   B *B>B 
B	B  
B.c                    [        U / SQS9n[        S U 5       5      (       a   eUR                  5        nUR                  USS SSS.S	9  S
S
S
5        US   R                  S:X  d   eUS   R
                  (       d   e[        U / SQS9n[        S U 5       5      (       a   e[        S U 5       5      (       a   eUR                  5        nUR                  USS SS0S	9  UR                  USS SS0S	9  S
S
S
5        US   R                  (       d   eUS   R
                  (       d   eUS   R
                  (       a   eUS   R                  (       a   e[        U / SQS9nUS   R                  S:X  d   eUR                  5        nUR                  USS SS0S	9  S
S
S
5        US   R                  S:X  d   eU S   R                  S:X  d   eg
! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N\= f)a  Test that retokenization also sets attributes on the lexeme if they're
lexical attributes. For example, if a user sets IS_STOP, it should mean that
"all tokens with that lexeme" are marked as a stop word, so the ambiguity
here is acceptable. Also see #2390.
r   rJ   c              3   8   #    U  H  oR                   v   M     g 7frA   rB   rD   r\   s     r"   rF   7test_doc_retokenizer_merge_lex_attrs.<locals>.<genexpr>       *c99crH   r   r3   r   T)r   rC   r   N)einszweir   r   c              3   8   #    U  H  oR                   v   M     g 7frA   )like_numr   s     r"   rF   r     s     +s!::srH   c              3   8   #    U  H  oR                   v   M     g 7frA   rB   r   s     r"   rF   r     r   rH   r   r   rC   r2   r   normr   )r   rL   r   r   r   rC   r   norm_ro   r    r!   s      r"   $test_doc_retokenizer_merge_lex_attrsr     s    h5
6C*c*****		[#a(Md*ST 
q6==M)))q6>>>
h8
9C+s+++++*c*****		[#a(:t*<=#a(9d*;< 
 q6???q6>>>1v~~1v
h8
9Cq6<<6!!!		[#a(63-8 
q6<<3F!!V++++ 
	 
	 
	s#   G+G%
G6
G"%
G36
Hc                    [        U / SQS9nUR                  5        nUR                  USS 5        UR                  USS 5        SSS5        [        U5      S:X  d   eUS   R                  S:X  d   eg! , (       d  f       N5= f)zoTest that the retokenizer automatically skips duplicate spans instead
of complaining about overlaps. See #3687.r   rJ   r   r3   Nr   )r   r   r   r   r   r   s      r"   test_retokenize_skip_duplicatesr     s{     h5
6C		[#a(##a(# 
 s8q==q6;;-'''	 
	s   )A55
Bc                     [        U / SQS9n[        R                  " [        5         UR	                  5        nUR                  USS 5        S S S 5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)Nr   rJ   r2   )r   r}   r~   r   r   r   r   s      r"   $test_retokenize_disallow_zero_lengthr     sX    
h5
6C	z	"^^c!Ah'  
#	" 
#	"s"   A/AA/
A,	(A//
A=c                 \   Sn/ SQnU " U5      n[        UR                  U Vs/ s H  oDR                  PM     snUS9n[        [	        UR
                  5      5      S:X  d   eUR                  5        nUR                  USS 5        S S S 5        [        [	        UR
                  5      5      S:X  d   e[        UR                  U Vs/ s H  oDR                  PM     snUS9n[        [	        UR
                  5      5      S:X  d   eUR                  5        nUR                  USS 5        S S S 5        US   R                  b   e[        UR                  U Vs/ s H  oDR                  PM     snUS9n[        [	        UR
                  5      5      S:X  d   eUR                  5        nUR                  USS SS	0S
9  S S S 5        [        [	        UR
                  5      5      S:X  d   eg s  snf ! , (       d  f       GNp= fs  snf ! , (       d  f       N= fs  snf ! , (       d  f       Ne= f)Nr   )r2   r   r   r   r2   r   r   r   )rK   sent_startsr3   r2   r.   r   
sent_startTr   )	r   r;   r   r   r(   r   r   r   is_sent_start)r   r   r   r[   r\   r    r!   s          r"   3test_doc_retokenize_merge_without_parse_keeps_sentsr     s   ;D*K$F fll6":6a666":
TCtCII1$$$		[#a(# 
tCII1$$$ fll6":6a666":
TCtCII1$$$		[#a(# 
q6''' fll6":6a666":
TCtCII1$$$		[#a(<*>? 
tCII1$$$% #;		
 #;		
 #;		s5   G+
2G0H
HH
(H0
G?
H
H+)r}   spacy.attrsr   spacy.tokensr   r   spacy.vocabr   r#   r,   r5   r7   r>   rN   r]   rp   r{   r   r   r   r   r   r   r   markparametrizer   r   r   r   r   r   r   r"   <module>r      s      # $42L+	%
%$)&X$#@B^(B',9$, +sCj3*sCjSTRU-VW5 X5,B((%r   