
    h                         S SK r S SKJrJr  S SKJr  S SKJr  S SKJ	r	  S SK
Jr  S SKJr  S SKJr  S S	KJr  S S
KJr  S SKJr  S r/ SQrS/ SQ/ SQS.4S/ SQ/ SQS.4/rS rS rS rS rS rS rg)    N)assert_almost_equalassert_equal)get_current_ops)util)MORPH)English)Language)
Morphology)make_tempdir)Doc)Examplec                      [        5       n U R                  S5      nUR                  S5        [        R                  " [
        5         UR                  S5        S S S 5        g ! , (       d  f       g = f)NmorphologizerzFeat=A	   )r	   add_pipe	add_labelpytestraises
ValueErrornlpr   s     a/home/james-whalen/.local/lib/python3.13/site-packages/spacy/tests/pipeline/test_morphologizer.pytest_label_typesr      sK    
*CLL1MH%	z	"" 
#	"	"s   A""
A0)Feat=NFeat=VFeat=JzI like green eggs)r   r   r   r   )NOUNVERBADJr   )morphsposzEat blue ham)r    r"   )r"   r   r"   c            	        ^ [        5       n U R                  SS5      nU R                  SS[        SS9S9n/ m0 n[         H%  nUR	                  U5        UR	                  U5        M'     [
         H>  nTR                  [        R                  " U R                  US   5      US   5      5        M@     U R                  U4S	 jS
9  UR                  R                  T Vs/ s H  ofR                  PM     sn5      u  px[        5       n	U	R                  UR!                  TU5      S   S   5      n
U	R                  UR!                  TU5      S   S   5      n[#        X-  S5        g s  snf )Nr   no_label_smoothinglabel_smoothingg?)r%   )configr      c                     > T $ N train_exampless   r   <lambda>&test_label_smoothing.<locals>.<lambda>4           get_examplesg+?)r	   r   dictTAGSr   
TRAIN_DATAappendr   	from_dictmake_doc
initializemodelbegin_update	predictedr   to_numpyget_lossr   )r   morph_no_lsmorph_lslossestagteg
tag_scoresbp_tag_scoresopsno_ls_gradsls_gradsr,   s               @r   test_label_smoothingrJ   &   sI   
*C,,0DEK||*43M  H NFc"3  g//QqT0BAaDIJ  NN 6N7 ( ; ; ./"/!J 
C,,{33NJOPQRSTUVK||H--njI!LQOPH.
; 	0s   E"c                      [        5       n U R                  S5        [        R                  " [        5         U R                  5         S S S 5        g ! , (       d  f       g = f)Nr   )r	   r   r   r   r   r9   )r   s    r   test_no_labelrL   >   s7    
*CLL!	z	" 
#	"	"s   A
Ac            	         ^ [        5       n U R                  S5        / m[         H>  nTR                  [        R
                  " U R                  US   5      US   5      5        M@     U R                  U4S jS9  g )Nr   r   r'   c                     > T $ r)   r*   r+   s   r   r-   %test_implicit_label.<locals>.<lambda>K   r/   r0   r1   )r	   r   r5   r6   r   r7   r8   r9   )r   rC   r,   s     @r   test_implicit_labelrP   E   s`    
*CLL!Ng//QqT0BAaDIJ NN 6N7r0   c                     [        5       n U R                  S5      nUR                  S[        R                  -   S-   5        UR                  S[        R                  -   S-   5        U R                  5         [        R                  " [        5         UR                  S[        R                  -   S-   5        S S S 5        g ! , (       d  f       g = f)Nr   POSr   r   r   )	r	   r   r   r
   	FIELD_SEPr9   r   r   r   r   s     r   test_no_resizerT   N   s    
*CLL1MEJ$8$886ABEJ$8$886ABNN	z	"
(<(< <u DE 
#	"	"s   &B??
Cc            	      n  ^ [        5       n U R                  S5      nUR                  S[        R                  -   S-   5        / m[
         H>  nTR                  [        R                  " U R                  US   5      US   5      5        M@     U R                  5         U R                  U4S jS9  [        R                  " [        5         U R                  S S9  S S S 5        [        R                  " [        5         U R                  TS9  S S S 5        g ! , (       d  f       NA= f! , (       d  f       g = f)	Nr   rR   r   r   r'   c                     > T $ r)   r*   r+   s   r   r-   *test_initialize_examples.<locals>.<lambda>b   r/   r0   r1   c                      g r)   r*   r*   r0   r   r-   rW   d   s    Dr0   )r	   r   r   r
   rS   r5   r6   r   r7   r8   r9   r   r   	TypeError)r   r   rC   r,   s      @r   test_initialize_examplesrZ   Y   s    
*CLL1MEJ$8$886ABNg//QqT0BAaDIJ  NNNN 6N7	y	!L1 
"	y	!N3 
"	! 
"	!	!	!s   	D<D&
D#&
D4c            	        ^ [        5       n U R                  S5        / m[         H>  nTR                  [        R
                  " U R                  US   5      US   5      5        M@     U R                  U4S jS9n[        S5       H  n0 nU R                  TX$S9  M     WS   S:  d   eS	nU " U5      n/ S
Qn/ SQnU V	s/ s H  n	[        U	R                  5      PM     sn	U:X  d   eU V	s/ s H  oR                  PM     sn	U:X  d   e[        5        n
U R                  U
5        [        R                   " U
5      nU" U5      nU V	s/ s H  n	[        U	R                  5      PM     sn	U:X  d   eU V	s/ s H  oR                  PM     sn	U:X  d   e S S S 5        / SQnU R#                  U5       Vs/ s H  ofR%                  [&        /5      PM     nnU R#                  U5       Vs/ s H  ofR%                  [&        /5      PM     nnU Vs/ s H  nU " U5      PM     sn Vs/ s H  ofR%                  [&        /5      PM     nn[)        X5        [)        UU5        U R+                  S5        U R                  S5        T H  nUR,                   H
  nSUl        M     M     U R                  U4S jS9n[        S5       H  n0 nU R                  TX$S9  M     US   S:  d   eS	nU " U5      n/ S
Qn/ SQnU V	s/ s H  n	[        U	R                  5      PM     sn	U:X  d   eU V	s/ s H  oR                  PM     sn	U:X  d   e/ SQn[/        U R0                  / SQUS9nU V	s/ s H  n	[        U	R                  5      PM     nn	U V	s/ s H  oR                  PM     nn	U R3                  S5      nSUR4                  S'   U" U5      nU V	s/ s H  n	[        U	R                  5      PM     sn	U:X  d   eU V	s/ s H  oR                  PM     sn	U:X  d   eSUR4                  S'   SUR4                  S'   [/        U R0                  SS/SS/S9nU" U5      nU V	s/ s H  n	[        U	R                  5      PM     sn	SS/:X  d   eSUR4                  S'   SUR4                  S'   [/        U R0                  SS/SS/S9nU" U5      nU V	s/ s H  n	[        U	R                  5      PM     sn	SS/:X  d   eSUR4                  S'   SUR4                  S'   [/        U R0                  SS/SS/S9nU" U5      nU V	s/ s H  n	[        U	R                  5      PM     sn	SS/:X  d   eU R+                  S5        U R                  S5        T HF  nUR,                   H3  nUR6                  S:X  a  S Ul        OSUl        UR9                  S 5        M5     MH     U R                  U4S! jS9nU R3                  S5      R:                  c   e[        S5       H  n0 nU R                  TX$S9  M     US   S:  d   eS	nU " U5      n/ SQn/ S"QnU V	s/ s H  n	[        U	R                  5      PM     sn	U:X  d   eU V	s/ s H  oR                  PM     sn	U:X  d   eg s  sn	f s  sn	f s  sn	f s  sn	f ! , (       d  f       GN= fs  snf s  snf s  snf s  snf s  sn	f s  sn	f s  sn	f s  sn	f s  sn	f s  sn	f s  sn	f s  sn	f s  sn	f s  sn	f s  sn	f )#Nr   r   r'   c                     > T $ r)   r*   r+   s   r   r-   %test_overfitting_IO.<locals>.<lambda>p       Nr0   r1   2   )sgdrA   gh㈵>zI like blue ham)r   r   r"   r"   )r   r   r   r"   )zJust a sentence.z$Then one more sentence about London.zHere is another one.zI like London.r"   c                     > T $ r)   r*   r+   s   r   r-   r]      r^   r0   )r"   r"   r"   r"   )r   r   _)bluehamlike)wordsr    F	overwriteTextendIre   zFeat=A|That=A|This=AzFeat=N|That=A|This=Ar   zThat=BzFeat=V|That=Br   rd   r   c                     > T $ r)   r*   r+   s   r   r-   r]      r^   r0   )r   r   r   r   )r   r   r5   r6   r   r7   r8   r9   rangeupdatestrmorphpos_r   to_diskr   load_model_from_pathpipeto_arrayr   r   remove_pipe	referencer   vocabget_pipecfgtext	set_morphlabels)r   inst	optimizerirA   	test_textdocgold_morphsgold_pos_tagsrC   tmp_dirnlp2doc2textsbatch_deps_1batch_deps_2ry   no_batch_depsexampletokenr    orig_morphsorig_pos_tagsr   r,   s                           @r   test_overfitting_IOr   i   sp   
)CLL!Ng//T!W0EtAwOP ,BCI2Y

>y
@  /"W,,, "I
i.C.K/M"%&#QCL#&+555 CqFFC M111 
7G((1I&*+dAGGd+{::: $%1%666 
E 69XXe_E_cLL%)_LE58XXe_E_cLL%)_LEMR6SUTs4yU6ST6Ss\\5'*6SMT,}- OOO$LL!!&&EEJ ' " ,BCI2Y

>y
@  /"W,,, "I
i.C.K$M"%&#QCL#&+555 CqFFC M111 'F
cii6v
FC),-A3qww<K-%()SVVSM)LL1M &+Mk"

C"%&#QCL#&+555 CqFFC M111 &*Mk""&Mh
ciiV}6Lb5Q
RC

C"%&#QCL#&+A8*LLLL &+Mk""&Mh
ciiV}6Lh5W
XC

C"%&#QCL#&+A?*SSSS &*Mk""'Mh
ciiV}6Lb5Q
RC

C"%&#QCL#&8X*>>>> OOO$LL!!&&EzzU"#

OOD! ' " ,BCI<<(//;;;2Y

>y
@  /"W,,, "I
i.C"K4M"%&#QCL#&+555 CqFFC M111M '  ,% 
 FE6ST* '  .) '  ' ' '0 ' s   ?Z+Z4ZZ&Z3Z	Z1Z-%Z2
Z7!Z<6["[[[[3[[[$.[)<[.([3
Z
Z*)r   numpy.testingr   r   	thinc.apir   spacyr   spacy.attrsr   spacy.lang.enr   spacy.languager	   spacy.morphologyr
   spacy.tests.utilr   spacy.tokensr   spacy.trainingr   r   r4   r5   rJ   rL   rP   rT   rZ   r   r*   r0   r   <module>r      s~     ; %   ! # ' )  "# & 	>2	
  2?KL

<08F4 y2r0   