
    h(                     x   S SK r S SKJr  S SKrS SKJr  S SKJr  S SK	J
r
  S SKJr  S SKJr  S SKJr  S SKJr  S S	KJr  S
S/ SQ04SS/ SQ04/rS
S/ SQ04S/ SQ/ SQS.4/rS rS r\R2                  R5                  SS5      S 5       r\R2                  R5                  SS5      S 5       r\R2                  R5                  SS5      S 5       rS rS rS r S r!S r"\" \RF                  " 5       \RF                  " 5       5      S  5       r$\" \RF                  " S!S"9\RF                  " S!S"95      S# 5       r%S$ r&S% r'g)&    N)given)util)English)Language)	EditTrees)StringStore)Example)make_tempdirzShe likes green eggslemmas)shelikegreeneggzEat blue ham)eatblueham) r   r   r   zHe hates green eggs)Hehatesr   eggs)r   r   er   r   )wordsr   c            	        ^ [        5       n U R                  S5      n/ m[         H>  nTR                  [        R
                  " U R                  US   5      US   5      5        M@     U R                  U4S jS9  [        R                  " [        5         U R                  S S9  S S S 5        [        R                  " [        5         U R                  U4S jS9  S S S 5        [        R                  " [        5         U R                  S S9  S S S 5        [        R                  " [        5         U R                  TS9  S S S 5        g ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       Nc= f! , (       d  f       g = f)	Ntrainable_lemmatizerr      c                     > T $ N train_exampless   h/home/james-whalen/.local/lib/python3.13/site-packages/spacy/tests/pipeline/test_edit_tree_lemmatizer.py<lambda>*test_initialize_examples.<locals>.<lambda>)           get_examplesc                      g r   r   r   r&   r"   r#   r$   +   s    Dr&   c                     > T S   $ )Nr   r   r    s   r"   r#   r$   -   s
    N1,=r&   c                      / $ r   r   r   r&   r"   r#   r$   /   s    Br&   )r   add_pipe
TRAIN_DATAappendr	   	from_dictmake_doc
initializepytestraises	TypeError)nlp
lemmatizertr!   s      @r"   test_initialize_examplesr8   "   s   
*C45JNg//QqT0BAaDIJ  NN 6N7	y	!L1 
"	y	!$=> 
"	y	!J/ 
"	y	!N3 
"	! 
"	!	!	!	!	!	!	!s0   E	E=E+0E<	
E
E(+
E9<
F
c                    ^ [        5       n U R                  S5      nSUl        / m[         H>  nTR	                  [
        R                  " U R                  US   5      US   5      5        M@     U R                  U4S jS9  [        5       nUR                  S5      nUR                  U4S jUR                  S9  UR                  SSSS	S
.:X  d   eUR                  SSS.SSSSS.SSS.SSSSS.SSSSS.SSS.SSSSS./S
S.:X  d   eg )Nr   r   r   c                     > T $ r   r   r    s   r"   r#   -test_initialize_from_labels.<locals>.<lambda>;   r%   r&   r'   c                     > T S S $ )Nr   r   r    s   r"   r#   r;   C   s    ^BQ/r&   r(   labels      )r   r@         Ss)origsubstl    )
prefix_len
suffix_lenprefix_treesuffix_treer   Er      )treesr>   )r   r,   min_tree_freqr-   r.   r	   r/   r0   r1   
label_data
tree2labelr5   r6   r7   nlp2lemmatizer2r!   s        @r"   test_initialize_from_labelsrT   4   sJ   
*C45J JNg//QqT0BAaDIJ NN 6N7:D-- 67K 0$$   !!qQ1%====!!3' )	 2&) 	  ))	 3' )	-
: =&   r&   top_k)r   rL      c           	        ^ SSSSS.04SSSSS.04/n[        5       nUR                  SSU 0S	9  UR                  S
5        / mU H>  nTR                  [        R                  " UR                  US   5      US   5      5        M@     [        R                  " [        5         UR                  U4S jS9  S S S 5        g ! , (       d  f       g = f)NzI'm so happy.catsg      ?g        )POSITIVENEGATIVEzI'm so angryr   rU   configtextcatr   r   c                     > T $ r   r   r    s   r"   r#   test_no_data.<locals>.<lambda>y   s    Nr&   r'   )
r   r,   r.   r	   r/   r0   r2   r3   
ValueErrorr1   )rU   TEXTCAT_DATAr5   r7   r!   s       @r"   test_no_datarb   i   s     
6#EFG	&s"DEFL )CLL'%0@LALLNg//QqT0BAaDIJ  
z	"$:; 
#	"	"s   !B>>
Cc           	        ^ [        5       nUR                  SSU 0S9nSUl        / m[         H>  nTR	                  [
        R                  " UR                  US   5      US   5      5        M@     UR                  U4S jS9n[        S5       H  n0 nUR                  TXFS	9  M     WS   S
:  d   eSnU" U5      nUS   R                  S:X  d   eUS   R                  S:X  d   eUR                  T V	s/ s H  oR                  PM     sn	SS9u  pUR                  TU
5      u  pUR                  R                  R                   nUR#                  US   S   5      S:X  d   eUR#                  US   S   5      S:X  d   eUR#                  US   S   5      S:X  d   eUR#                  US   S   5      S:X  d   eUR#                  US   S   5      S:X  d   eg s  sn	f )Nr   rU   r[   r   r   c                     > T $ r   r   r    s   r"   r#   &test_incomplete_data.<locals>.<lambda>       Nr&   r'   2   sgdlossesh㈵>She likes blue eggsr   r?   r   T)is_trainr@   )r   r,   rN   PARTIAL_DATAr.   r	   r/   r0   r1   rangeupdatelemma_model	predictedget_lossopsxpcount_nonzero)rU   r5   r6   r7   	optimizerirj   	test_textdocegscores_dXrv   r!   s                 @r"   test_incomplete_datar   |   s    )C4gu=MNJ JNg//QqT0BAaDIJ ,BCI2Y

>y
@  ()G333 &I
i.Cq6==F"""q6==F"""   !H2,,!HSW XIF7EA						 	 B BqE!H%***BqE!H%***BqE!H%***BqE!H%*** BqE!H%*** "Is   4Gc           	        ^ [        5       nUR                  SSU 0S9nSUl        / m[         H>  nTR	                  [
        R                  " UR                  US   5      US   5      5        M@     UR                  U4S jS9n[        S5       H  n0 nUR                  TXFS	9  M     WS   S
:  d   eSnU" U5      nUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   e[        R                  " 5        n	UR                  U	5        [        R                  " U	5      n
U
" U5      nUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   e S S S 5        UR!                  5       n[        5       nUR                  SSU 0S9  UR#                  U5        U" U5      nUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   e[$        R&                  " U5      n[$        R(                  " U5      nU" U5      nUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eUS   R                  S:X  d   eg ! , (       d  f       GN1= f)Nr   rU   r[   r   r   c                     > T $ r   r   r    s   r"   r#   %test_overfitting_IO.<locals>.<lambda>   rf   r&   r'   rg   rh   rk   rl   r   r   r?   r   r@   r   )r   r,   rN   r-   r.   r	   r/   r0   r1   ro   rp   rq   r   r
   to_diskload_model_from_pathto_bytes
from_bytespickledumpsloads)rU   r5   r6   r7   rx   ry   rj   rz   r{   tmp_dirrR   doc2	nlp_bytesnlp3doc3nlp4doc4r!   s                    @r"   test_overfitting_IOr      s   
)C4gu=MNJ JNg//QqT0BAaDIJ  ,BCI2Y

>y
@  ()G333%I
i.Cq6==E!!!q6==F"""q6==F"""q6==E!!! 
			G((1IAw~~&&&Aw~~'''Aw~~'''Aw~~&&& 
 I9DMM('51AMBOOI	?D7>>U"""7>>V###7>>V###7>>U""" S!I<<	"D	?D7>>U"""7>>V###7>>V###7>>U"""7 
	s   $BK
K"c                      [        5       n U R                  S5        [        R                  " [        5         U R                  5         S S S 5        g ! , (       d  f       g = f)Nr   )r   r,   r2   r3   r`   r1   )r5   s    r"   test_lemmatizer_requires_labelsr      s8    
)CLL'(	z	" 
#	"	"s   A
Ac            	        ^ [        5       n U R                  S5      nSUl        / m[         H>  nTR	                  [
        R                  " U R                  US   5      US   5      5        M@     U R                  U4S jS9  [        5       nUR                  S5      nUR                  U4S jUR                  S9  UR                  UR                  :X  d   eUR                  R                  5       UR                  R                  5       :X  d   eg )Nr   r   r   c                     > T $ r   r   r    s   r"   r#   ,test_lemmatizer_label_data.<locals>.<lambda>   r%   r&   r'   c                     > T $ r   r   r    s   r"   r#   r      s    ^r&   r=   )r   r,   rN   r-   r.   r	   r/   r0   r1   rO   r>   rM   r   rQ   s        @r"   test_lemmatizer_label_datar      s    
)C45J JNg//QqT0BAaDIJ  NN 6N79D-- 67K+J4I4I  
  2 2222$$&+*;*;*D*D*FFFFr&   c                      [        5       n [        U 5      nUR                  SS5      nUR                  U5      S:X  d   eUR                  SS5      nUR                  U5      S:X  d   eg )Ndeeltdelenz*(m 0 3 () (m 0 2 (s '' 'l') (s 'lt' 'n')))gedeeldz3(m 2 3 (s 'ge' '') (m 0 2 (s '' 'l') (s 'ld' 'n'))))r   r   addtree_to_str)stringsrM   trees      r"   
test_dutchr      si    mGgE99Wg&DT"&RRRR99Y(D$#XXXr&   c                      [        5       n [        U 5      nUR                  SS5        UR                  SS5        UR                  5       n[        U 5      nUR	                  U5        [        U5      [        U5      :X  d   e[        [        U5      5       H)  nUR                  U5      UR                  U5      :X  a  M)   e   UR                  SS5        UR                  SS5        [        U5      [        U5      :X  d   eg )Nr   r   r   )r   r   r   r   r   lenro   r   )r   rM   btrees2ry   s        r"   test_from_to_bytesr      s    mGgE	IIgw	IIi!AwF
a u:V$$$3u:  #v'9'9!'<<<<  JJw 
JJy'"u:V$$$r&   c                  T   [        5       n [        U 5      nUR                  SS5        UR                  SS5        [        U 5      n[        5        nUS-  nUR	                  U5        UR                  U5      nS S S 5        [        U5      [        U5      :X  d   e[        [        U5      5       H)  nUR                  U5      UR                  U5      :X  a  M)   e   UR                  SS5        UR                  SS5        [        U5      [        U5      :X  d   eg ! , (       d  f       N= f)Nr   r   r   zedit_trees.bin)	r   r   r   r
   r   	from_diskr   ro   r   )r   rM   r   temp_dir
trees_filery   s         r"   test_from_to_diskr     s    mGgE	IIgw	IIi!wF	8 00
j!!!*- 
 u:V$$$3u:  #v'9'9!'<<<<  JJw 
JJy'"u:V$$$ 
s   (D
D'c                 ~    [        5       n[        U5      nUR                  X5      nUR                  X@5      U:X  d   eg r   r   r   r   applyformlemmar   rM   r   s        r"   test_roundtripr   +  s8    mGgE99T!D;;t"e+++r&   ab)alphabetc                 ~    [        5       n[        U5      nUR                  X5      nUR                  X@5      U:X  d   eg r   r   r   s        r"   test_roundtrip_small_alphabetr   3  s:     mGgE99T!D;;t"e+++r&   c                      [        5       n [        U 5      nUR                  SS5      nUR                  US5      S :X  d   eUR                  US5      S :X  d   eg )Nr   r   deeldder   )r   rM   tree3s      r"   test_unapplicable_treesr   <  sY    mGgEIIgw'E ;;ug&$... ;;ud#t+++r&   c                      [        5       n [        U 5      nUR                  SS5      nUR                  SS5      nX#:X  d   eg )Nxyzr   )r   r   r   )r   rM   	no_changeemptys       r"   test_empty_stringsr   H  s?    mGgE		%'IIIb"Er&   )(r   hypothesis.strategies
strategiesstr2   
hypothesisr   spacyr   spacy.lang.enr   spacy.languager   .spacy.pipeline._edit_tree_internals.edit_treesr   spacy.stringsr   spacy.trainingr	   
spacy.utilr
   r-   rn   r8   rT   markparametrizerb   r   r   r   r   r   r   r   textr   r   r   r   r   r&   r"   <module>r      sk    "    ! # D % " # h(GHIh 678
 h(ABC 	93	
	4$2j *-< .<$ *- + . +F *-2# .2#jG*	%,%. rwwy"''), , rwwrww56, 7,	,r&   