
    hr                     %   S SK r S SKrS SKJrJr  S SKrS SKrS SKJrJ	r	  S SK
JrJr  S SKJr  S SKJr  S SKJrJrJrJrJrJrJrJr  S SKJrJrJr  S S	KJrJ r J!r!  S S
K"J#r#  S SK$J%r%J&r&  SSK'J(r(  SSK(J)r)J*r*  Sr+\" \5      r,\" 5       r-\" SS9r.\-\,/r/\-/r0\(       a  \0Rc                  \" 5       5        \0\,/-   r2SS/r3SS/r4/ SQr5/ SQr6S r7\ (       a  \7" 5       r8O/ r8\Rr                  Ru                  S\\/5      S 5       r;\Rr                  Ru                  S\25      \Rr                  Ry                  S5      S 5       5       r=\Rr                  Ru                  S\25      S 5       r>\Rr                  Ru                  S\05      S 5       r?\Rr                  Ru                  S\05      S  5       r@\Rr                  Ru                  S\05      S! 5       rA\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      \Rr                  Ru                  S#SS$/5      S% 5       5       5       rB\Rr                  Ru                  S\05      \" \(RR                  " 5       S&9S' 5       5       rC\Rr                  Ru                  S\25      S( 5       rD\Rr                  Ru                  S\/5      S) 5       rE\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      \	" \+SS*9\" \(R                  " 5       S+9S, 5       5       5       5       rG\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      S- 5       5       rH\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9S. 5       5       5       rI\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S/ 5       5       rJ\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S0 5       5       rK\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S1 5       5       rL\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S2 5       5       rM\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S3 5       5       rN\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S4 5       5       rO\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      \	" \+SS*9\" \(RR                  " 5       S+9S5 5       5       5       5       rP\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S6 5       5       rQ\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S7 5       5       rR\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S8 5       5       rS\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      S9 5       5       rT\Rr                  R                  \(       + S:S;9\Rr                  Ru                  S<S=S/5      S> 5       5       rV\Rr                  R                  \(       + S:S;9\Rr                  Ru                  S<S=S/5      S? 5       5       rW\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      \	" \+SS*9\" \(RR                  " 5       S+9S@ 5       5       5       5       rX\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9SA 5       5       5       rY\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9SB 5       5       5       rZSC\#SD\#SE\[SF\\#\#4   4SG jr\\Rr                  R                  \ (       + SHS;9\Rr                  Ru                  S\25      \Rr                  Ru                  SE/ SIQ5      SJ 5       5       5       r]\Rr                  Ru                  SK/ \/Q\.P5      SL 5       r^\Rr                  Ru                  SK/ \/Q\.P5      SM 5       r_\Rr                  Ru                  SK\/5      \	" \+SS*9\" \(RR                  " 5       S+9SN 5       5       5       r`\Rr                  Ru                  S\25      \Rr                  Ru                  S"\3\4-   5      SO 5       5       ra\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SP 5       5       rb\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SQ 5       5       rc\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SR 5       5       rd\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SS 5       5       re\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      ST 5       5       rf\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SU 5       5       rg\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SV 5       5       rh\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SW 5       5       ri\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SX 5       5       rj\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SY 5       5       rk\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      SZ 5       5       rl\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      \Rr                  Ru                  S[\55      S\ 5       5       5       rm\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      \Rr                  Ru                  S[\55      S] 5       5       5       rn\Rr                  Ru                  S\25      \Rr                  Ru                  S"\35      \Rr                  Ru                  S^\65      S_ 5       5       5       ro\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9S` 5       5       5       rp\Rr                  Ru                  S\05      \Rr                  Ru                  S"\35      \Rr                  Ru                  S/ SaQ5      Sb 5       5       5       rq\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Sc 5       5       5       rr\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Sd 5       5       5       rs\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Se 5       5       5       rt\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Sf 5       5       5       ru\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Sg 5       5       5       rv\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Sh 5       5       5       rw\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Si 5       5       5       rx\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Sj 5       5       5       ry\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Sk 5       5       5       rz\Rr                  Ru                  S\25      \	" \+SS*9\" \(RR                  " 5       S+9Sl 5       5       5       r{SSm jr|\Sn 5       r}\Rr                  Ru                  S\05      \Rr                  Ru                  So/ SpQ5      Sq 5       5       r~\Rr                  R                  SrS;9\Rr                  Ru                  S\05      \	" \+SS*9\" \}" 5       Ss9St 5       5       5       5       rSu rSv rSw rSx rSy r\Rr                  R                  \ (       + SHS;9\Rr                  R                  \!\" Sz5      :  S{S;9\Rr                  Ru                  S\25      \Rr                  Ru                  S"SS/5      \Rr                  Ru                  S|\85      \	" \+SS*9\" \(GR                  " S}S~S9\(GR                  " SS=S9S9S 5       5       5       5       5       5       5       r\Rr                  Ru                  S\25      \	" \+SS*9\" \(GR                  " SSS9S9S 5       5       5       r\Rr                  Ru                  S\25      \Rr                  Ru                  SS5      \	" \+SS*9\" \(GR                  " SSS9S9S 5       5       5       5       r\Rr                  R                  \(       + S:S;9S 5       r\Rr                  Ru                  S\25      S 5       rg)    N)Tuplecast)givensettings)	compositeintegers)assert_allclose)Version)LSTMCupyOpsNumpyOpsOpsfix_random_seedget_current_opsget_opsuse_ops)KERNELSKERNELS_LISTcompile_mmh)has_cupy_gpu	has_torchtorch_version)Floats2d)torch2xpxp2torch   )
strategies)	arrays_BIndarrays_of_shape
   T)use_blisfloat32float64int32int64)reduce_firstreduce_last
reduce_maxreduce_mean
reduce_sum))r&   T)r'   T)r(   T)r)   F)r*   Fc                     ^^^ SS K mSS KmU4S jn U4S jnU4S jmU4S jnU4S jnU4S jnU4S jnU4S	 jnU4S
 jnS nUU4S jn	U4S jn
SU 4SU4ST4SU4SU4SU4SU4SU4SU4SU	4SU
4SU4/$ )Nr   c                 N   > TR                   R                  R                  U 5      $ N)nn
functionalreluxtorchs    W/home/james-whalen/.local/lib/python3.13/site-packages/thinc/tests/backends/test_ops.py
torch_relu(create_pytorch_funcs.<locals>.torch_relu@       xx""''**    c                 N   > TR                   R                  R                  U 5      $ r-   )r.   r/   relu6r1   s    r4   torch_relu_k*create_pytorch_funcs.<locals>.torch_relu_kC   s    xx""((++r8   c                 6   > TR                  U S-  S-   SS5      $ )N皙?      ?r      )clipr1   s    r4   torch_hard_sigmoid0create_pytorch_funcs.<locals>.torch_hard_sigmoidF   s    zz!c'C-A..r8   c                 N   > TR                   R                  R                  U 5      $ r-   )r.   r/   hardtanhr1   s    r4   torch_hard_tanh-create_pytorch_funcs.<locals>.torch_hard_tanhI   s    xx""++A..r8   c                 N   > TR                   R                  R                  U 5      $ r-   )r.   r/   mishr1   s    r4   
torch_mish(create_pytorch_funcs.<locals>.torch_mishL   r7   r8   c                 N   > TR                   R                  R                  U 5      $ r-   )r.   r/   silur1   s    r4   torch_swish)create_pytorch_funcs.<locals>.torch_swishO   r7   r8   c                    > U T" U 5      -  $ r-    )r2   rB   s    r4   torch_hard_swish.create_pytorch_funcs.<locals>.torch_hard_swishR   s    %a(((r8   c                 N   > TR                   R                  R                  U 5      $ r-   )r.   r/   	hardswishr1   s    r4   torch_hard_swish_mobilenet8create_pytorch_funcs.<locals>.torch_hard_swish_mobilenetU   s    xx"",,Q//r8   c                 &   > TR                  U 5      $ r-   )sigmoidr1   s    r4   torch_sigmoid+create_pytorch_funcs.<locals>.torch_sigmoidX   s    }}Qr8   c                 D    SU -  U SX -  -   R                  5       -  S-   -  $ )Nr?   r@   )sqrtr2   s    r4   
torch_dish(create_pytorch_funcs.<locals>.torch_dish[   s*    Qw!q15y..001455r8   c                    > SU -  STR                  TR                  STR                  -  5      U STR                  U S5      -  -   -  5      -   -  $ )Nr?         ?       @gHm?      @)tanhr]   pipow)r2   mathr3   s    r4   torch_gelu_approx/create_pytorch_funcs.<locals>.torch_gelu_approx_   s\     **IIcDGGm,HuyyC?P4P0PQ		
r8   c                 N   > TR                   R                  R                  U 5      $ r-   )r.   r/   gelur1   s    r4   
torch_gelu(create_pytorch_funcs.<locals>.torch_geluk   r7   r8   r0   relu_khard_sigmoid	hard_tanhrI   swish
hard_swishhard_swish_mobilenetdishgelu_approxrl   rY   )rh   r3   )r5   r;   rF   rJ   rN   rR   rV   rZ   r_   ri   rm   rh   r3   rB   s              @@@r4   create_pytorch_funcsrw   ;   s    +,//++)0 6

+ 
	< 	+,	o&		+	'(	!;<		)*		M" r8   opc                    [        U 5       Vs/ s H  oR                  S5      (       a  M  UPM     nnU GH  n[        [        U5      (       d   e[	        X5      n[        US5      (       d  M9  [
        R                  " U5      nUR                   Vs/ s H  ofPM     snSS n[
        R                  " [	        [        U5      5      n[        U5        UR                   Vs/ s H  ofPM     snSS n	Xy:X  d   U5       eUR                  R                  5        Vs/ s H  ofR                  PM     snSS n
UR                  R                  5        Vs/ s H  ofR                  PM     snSS nX:X  d   U5       eUR                  R                  5        Vs/ s H  ofR                  PM     snSS nUR                  R                  5        Vs/ s H  ofR                  PM     snSS n[        [        X5      5       HI  u  nu  nnU[
        R                  R                  :w  d  M)  U[
        R                  R                  :w  d  MI  MK     GM     gs  snf s  snf s  snf s  snf s  snf s  snf s  snf )zTest that specific ops don't define any methods that are not on the
Ops base class and that all ops methods define the exact same arguments.___call__r@   N)dir
startswithhasattrr   getattrinspect	signature
parametersprintvaluesdefault
annotation	enumeratezip	Parameterempty)rx   mattrsattrmethodsigpparamsbase_sigbase_paramsdefaultsbase_defaultsannotsbase_annotsip1p2s                    r4   test_ops_consistencyr      s    B91||C'8QE9sD!!!!"6:&&##F+C!$0Aa04F((d);<H(O&.&9&9:&91&9:12>K(.$.(+.>>+@+@+BC+Ba		+BCABGH080C0C0J0J0LM0L1YY0LMabQM,2d2,,/NN,A,A,CD,Cqll,CDQRHF191D1D1K1K1MN1MA<<1MNqrRK(V)AB8B**000R7;L;L;R;R5R
   C!  : 1 ;CM ENs.   III III I%
I*opszignore::RuntimeWarningc                    U R                   R                  SSS9nU R                   R                  SSS9nU R                  XXSSSS5        [        R                  " [
        5         U R                  X!XSSSS5        S S S 5        [        R                  " [
        5         U R                  XXSSSS5        S S S 5        [        R                  " [
        5         U R                  XX!SSSS5        S S S 5        [        R                  " [
        5         U R                  XXSSSS5        S S S 5        g ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       Nj= f! , (       d  f       g = f)Nr@   fdtyper           )xpzerosadampytestraises
ValueError)r   onetwos      r4   test_adam_incorrect_inputsr      s    &&,,q,
$C
&&,,q,
$CHHSsc34	z	"3S#sC8 
#	z	"3S#sC8 
#	z	"3S#sC8 
#	z	"3S#sC8 
#	" 
#	"	"	"	"	"	"	"s0   %D0EEE#0
D>
E
E #
E1c                    U R                   U R                  U R                  U R                  4n[	        U5       H  u  p#SUS-   -  nU" U6 nUR
                  [        R                  :X  d   eUR                  [        U5      :X  d   eU R                  U5      nUR
                  [        R                  :X  d   eUR                  [        U5      :X  a  M   e   U R                  U R                  U R                  U R                  4n[	        U5       H  u  p#SUS-   -  nU" U6 nUR
                  [        R                  :X  d   eUR                  [        U5      :X  d   eU R!                  U5      nUR
                  [        R                  :X  d   eUR                  [        U5      :X  a  M   e   U R#                  S5      R                  S:X  d   eg )N)r@   r@   )alloc1falloc2falloc3falloc4fr   r   numpyr"   ndimlenalloc_falloc1ialloc2ialloc3ialloc4ir$   alloc_ialloc)r   float_methodsr   r   shapearrint_methodss          r4   
test_allocr      sx   [[#++s{{CKKHM}-	AenyyEMM)))xx3u:%%%kk% yyEMM)))xx3u:%%% . ;;S[[#++FK{+	AenyyEKK'''xx3u:%%%kk% yyEKK'''xx3u:%%% , 99Q<!!!r8   c                    U R                  SSS9nU R                  US5      nUR                  S:X  d   eUR                  S:X  d   e[	        [        U5      5       H.  n[	        UR                  S   5       H  nX#U4   S:w  a  M   e   M0     g )N   uint64r   r   )r      uint32r@   )r   hashr   r   ranger   )r   idskeysr   js        r4   test_hash_gives_distinct_keysr      s    
++ax+
(C88CD::::!!!3s8_tzz!}%A1:?"? & r8   c                     SnSnU R                  X5      nUS::  a  X3S:H     R                  5       (       d   eg X3S:g     R                  5       (       d   eg )Nr   r   r   rb   )get_dropout_maskallr   r   dropmasks       r4   test_get_dropout_emptyr      s[    ED,Ds{CK $$&&&&CK $$&&&&r8   c                     SnSnU R                  X5      nUS:  R                  5       (       d   eUS:H  R                  5       (       d   eUR                  U:X  d   eg )N)   r   r?   rb   r   )r   anyr   r   s       r4   test_get_dropout_not_emptyr      s`    ED,D3JCK::r8   r   index_dtyper   c                    U R                   R                  SUS9R                  SS5      nU R                   R                  SS/SS/SS//US9nU R	                  X45      nU R                   R
                  R                  U/ SQ/ S	Q/ S
Q/5        g )N   r   r      r   r   r@   )      @       @      $@)      (@g      ,@      0@)rd         @      @)r   arangereshapearray
gather_addtestingr	   )r   r   r   tableindicesgathereds         r4   test_gather_addr      s     FFMM"EM*221a8EffllQFQFQF3;lGG~~e-HFFNN""#%7Ir8   )r   c                 $   U R                  U5      nU R                  R                  SSS9R                  SS5      UR                  S   -  nU R                  R
                  R                  U R                  X5      X   R                  S5      SS	9  g )
Nd   r   r      r   r   r@   h㈵>atol)	asarrayr   r   r   r   r   r	   r   sumr   r   r   s      r4   test_gather_add_against_numpyr      s|     KKEffmmCsm+33B:U[[^KGFFNN""u&1 # r8   c                 (   U R                   R                  SSS9R                  SS5      nU R                   R                  SS/SS/S	S//S
S9n[        R
                  " [        5         U R                  X5        S S S 5        g ! , (       d  f       g = f)Nr   r   r   r   r   r   r   r@   r   r   )r   r   r   r   r   r   
IndexErrorr   r   s      r4   test_gather_add_oob_raisesr     sr    FFMM"CM(00A6EffllQFQFQF33l?G	z	"u& 
#	"	"s   (B
Bc                 "   U R                  S/S/S/S//SS9nU R                  US5      n[        US5      (       a  UR                  5       n[	        US	   / S
Q5        [	        US   / SQ5        [	        US   / SQ5        [	        US   / SQ5        g )Nrb   rd         @r   r"   r   r@   getr   )r   rb   rd   )rb   rd   r   r   )rd   r   r   r   )r   r   r   )r   seq2colr~   r   r	   )r   seqcolss      r4   test_seq2col_window_one_smallr   
  s    
++usecUQC0	+
BC;;sADtUxxzDG_-DG_-DG_-DG_-r8   )max_examplesdeadlineXc           	         U R                  X!S9nUR                  SS9R                  U5      nU R                  U5      u  pEUR                  U:X  d   eU R
                  R                  R                  X4SSS9  U R
                  R                  R                  U R
                  R                  X R
                  R                  US5      SS9U R
                  R                  US5      SS9  g )Nr   axisMbP?rtolr   g|=r   )
r   maxastypemaxoutr   r   r   r	   take_along_axisexpand_dims)r   r   r  expected_bestpredicted_bestwhichs         r4   test_maxoutr    s    
 	A#AEErEN))%0MJJqMN5(((FFNN""E #  FFNN""q&&"4"4UB"?bI="- # r8   c           	         U R                  U R                  SS/SS//US9U R                  SS/SS//5      S	5      nUR                  U:X  d   eU R                  R
                  R                  U/ S
Q/ SQ// SQ/ SQ//5        [        R                  " [        5         U R                  U R                  SS/SS//5      U R                  SS/S	S//5      S	5        S S S 5        g ! , (       d  f       g = f)Nrb   rc   rd   r   r   r@   r   r   r   )r   rb   r   rc   r   r   r   r   rd   )r   r   r   )
backprop_maxout	asarray2f	asarray2ir   r   r   r	   r   r   r   r   r   dXs      r4   test_backprop_maxoutr  -  s     
		SzC:.e<1v1v&'	
B
 88uFFNN""

?	+o-OP
 
z	"MMC:Sz23S]]QFQPQFCS5TVW	
 
#	"	"s   >C$$
C2c                 @   U R                  U5      n[        5       nU R                  Ul        UR                  UR                  5      U-   nUR                  UR                  U5      SS9nU R                  USS9nU R                  R                  R                  XESSS9  g )Nr@   nWr  r   r  )r   r   r   r   r   r   r   r	   )r   r  base_opsbaseXtarget	predicteds         r4   test_seq2col_window_oner#  A  s     	AAuH&&HKNN177#a'Eh..u5!<FA!$IFFNN""65u"Mr8   c                    U R                   R                  R                  U R                  SUS9U R	                  U R                  SUS9SU R                   R                  SSS9S95        U R                   R                  R                  U R                  SUS9U R                  U R                  SUS9SU R                   R                  SSS9S95        U R                   R                  R                  U R                  SUS9U R	                  U R                  SUS9SU R                  S/5      S95        U R                   R                  R                  U R                  SUS9U R                  U R                  SUS9SU R                  S/5      S95        U R                   R                  R                  U R                  SUS9U R	                  U R                  SUS9SU R                  SS/5      S95        U R                   R                  R                  U R                  SUS9U R                  U R                  SUS9SU R                  SS/5      S95        g )N)r   r   r   r@   )r   r$   lengthsr   )r   r   r	   r   r   r   backprop_seq2col	asarray1ir   r   s     r4   test_seq2col_lengths_all_zeror*  N  s    FFNN""		&	&IIfEI*Asvv||DPW|7X 	 	
 FFNN""		&	&IIfEI*Asvv||DPW|7X 	 	
 FFNN""		&	&CIIfEI2As}}aS?QR
 FFNN""		&	&IIfEI*As}}aS7I 	 	
 FFNN""		&	&CIIfEI2As}}aQRV?TU
 FFNN""		&	&IIfEI*As}}aV7L 	 	
r8   c                    U R                  / SQ/ SQ/ SQ/ SQ/ SQ/US9nU R                  / SQ/ SQ/ S	Q/ S
Q/ SQ/US9nU R                  R                  R                  UU R	                  U R                  R                  SSUS9R                  SS5      SU R                  SS/5      S95        U R                  R                  R                  UU R                  USU R                  SS/5      S95        U R                  R                  R                  UU R	                  U R                  R                  SSUS9R                  SS5      SU R                  SS/5      S95        g )N	r   r   r   r@   r   r   r   r      	r@   r   r   r   r   r-        	   	r   r   r-  r/  r0  r1  r       r   	r/  r0  r1  r    r3  r            	r    r3  r   r5  r6  r7  r   r   r   r   r   r   r-  r   r7                 !   $         rA  rb   r   r   r   r@   r   r%  	r  r   r   r	   r   r   r   r(  r'  r   r   
cols_check
grad_checks       r4   $test_seq2col_lengths_zero_first_lastrK  {  s^    ''*--	
   	J 	L,lKSX  J
 FFNN""FFMM#t5M199!Q?MM1a&) 	 	
 FFNN""MM1a&) 	 	
 FFNN""FFMM#t5M199!Q?MM1a&) 	 	
r8   c                 l   U R                  / SQ/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/US9nU R                  / S	Q/ S
Q/ SQ/ SQ/ SQ/ SQ/ SQ/US9nU R                  R                  R                  UU R	                  U R                  R                  SSUS9R                  SS5      SU R                  / SQ5      S95        U R                  R                  R                  UU R                  USU R                  / SQ5      S95        U R                  R                  R                  UU R	                  U R                  R                  SSUS9R                  SS5      SU R                  / SQ5      S95        U R                  R                  R                  UU R                  USU R                  / SQ5      S95        g )Nr,  r.  r2  r4  r8  )	r   r   r         r;        r=  )	rM  rN  r;  rO  rP  r=  r   r   r   r   r9  r:  r<  r@  rD  )    "   rC  )&   (   *   rb   g      6@r/  r   r@   )r   r   r   r%  )r   r   r   r   rG  rH  s       r4   !test_seq2col_lengths_zero_betweenrV    s    ''*----	
   J 	
   J FFNN""FFMM#t5M199!Q?MM), 	 	
 FFNN""MM), 	 	
 FFNN""FFMM#t5M199!Q?MM,/ 	 	
 FFNN""MM,/ 	 	
r8   c           
      (   U R                   R                  SSUS9R                  SS5      nU R                  / SQ5      nU R	                  USUS9nU R                   R
                  R                  U R                  / S	Q/ S
Q/ SQ/ SQ/ SQ/US9U5        g )Nrb   r   r   r   r   r@   r   r@   r@   r%  )	r   r   r   r@   r   r   r   r   r   )	r   r   r   r   r   r-  r/  r0  r1  r2  )	r/  r0  r1  r    r3  r   r   r   r   )	r   r   r   r5  r6  r7  r   r   r   r   r   r   r(  r   r   r	   r  r   r   r  r&  r   s        r4   test_seq2col_window_one_lengthsr[    s     	c4u-55a;AmmI&G;;q!W;-DFFNN""++...  	 		
 	r8   c           
      (   U R                   R                  SSUS9R                  SS5      nU R                  / SQ5      nU R	                  USUS9nU R                   R
                  R                  U R                  / S	Q/ S
Q/ SQ/ SQ/ SQ/US9U5        g )Nrb   r   r   r   r   rX  r   r%  )r   r   r   r   r   r   r@   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r-  r/  r0  r1  r    r3  r   )r   r   r   r   r   r-  r/  r0  r1  r    r3  r   r   r   r   )r   r   r-  r/  r0  r1  r    r3  r   r   r   r   r   r   r   )r   r   r   r   r   r   r5  r6  r7  r   r   r   r   r   r   rY  rZ  s        r4   test_seq2col_window_two_lengthsr]    s     	c4u-55a;AmmI&G;;q!W;-DFFNN""=@@@@  	 		
 	r8   c                     U R                  / SQ/ SQ/ SQ/US9nS/S/S//nU R                  US5      n[        U[        R                  5      (       d  UR                  5       n[        XCS	S	S
9  g )N)r   r   r   )      r   rb   r  r   r_  rc   rb   r@   r  r  )r   r'  
isinstancer   ndarrayr   r	   r   r   r   expectedr   s        r4   &test_backprop_seq2col_window_one_smallrd    sr     ;;	*O<E  D u%H


tQ
'Cc5==))ggiCE:r8   c                 \   UR                   S   S-  (       a  g U R                  X!S9nU R                  R                  U5      R	                  5       S:  a  g [        5       nU R                  Ul        UR                  USS9nU R                  USS9n[        UR                   S   5       H[  nXF   R                  5       XV   R                  5       -
  nUS:  d  US:  d  M6  [        Xg5        [        XF   5        [        XV   5        M]     U R                  R                  R                  XES	S	S
9  g )Nr@   r   r   rA  r  r   g皙?r  r  )r   r   r   absr	  r   r'  r   r   r   r   r	   )r   r   r  r  r!  r"  rowdiffs           r4    test_backprop_seq2col_window_onerj  &  s    
 	wwqzA~A#A
vvzz!}b uH&&HK&&qQ&/F$$Q1$-IV\\!_%{ 9>#5#5#77$;$*#&+).! & FFNN""65u"Mr8   c           
      (   U R                   R                  SSSUS9R                  SS5      nU R                  / SQ5      nU R	                  USUS9nU R                   R
                  R                  U R                  / S	Q/ S
Q/ SQ/ SQ/ SQ/US9USS9  g )Nrf  gffffff@stepr   r   r1  rX  r@   r%  )g?r?   g333333?)g	@g333333@g@)gffffff@皙@g@)gffffff@g333333@r   )r   gffffff@g@r   ư>r   r   r   r   r(  r'  r   r	   r  r   r   d_yr&  d_seqss        r4   (test_backprop_seq2col_window_one_lengthsrt  =  s     &&--Ss%-
8
@
@A
FCmmI&G!!#q'!:FFFNN""  	 		
 	 # r8   c                 >   U R                  S/S/S/S//US9nU R                  US5      n[        U[        R                  5      (       d  UR                  5       n[        US   / SQ5        [        US	   / S
Q5        [        US   / SQ5        [        US   / SQ5        g )Nrb   rc   rd   r   r   r   r   r   r   rb   rc   rd   r@   r   rb   rc   rd   r   rb   rc   rd   r   r   r   rc   rd   r   r   r   )r   r   r`  r   ra  r   r	   )r   r   r   r   s       r4   test_seq2col_window_tworz  T  s     ++usecUQC0+
>C;;sADdEMM**xxzDG67DG67DG67DG67r8   c           
      *   U R                   R                  SSSUS9R                  SS5      nU R                  / SQ5      nU R	                  USUS9nU R                   R
                  R                  U R                  / S	Q/ S
Q/ SQ/ SQ/ SQ/US9U5        g )Nrf  gffffff@rl  r   r7  rX  r   r%  )gffffff?g?g?)gffffff$@g      %@g%@)g333333&@g&@gffffff'@)r   g(@g333333)@)g@g333333@rn  r   rp  rq  s        r4   (test_backprop_seq2col_window_two_lengthsr|  a  s     &&--Ss%-
8
@
@B
GCmmI&G!!#q'!:FFFNN"""""  	 		
 	r8   c                     U R                  / SQ/ SQ/ SQ/ SQ/US9nU R                  S/S/S/S//US9nU R                  US	5      nU R                  R                  R	                  XCS
S
S9  g )Nrv  rw  rx  ry  r   rd   r   r   r   r  r  )r   r'  r   r   r	   rb  s        r4    test_backprop_seq2col_window_twor~  w  s     ;;%%%%		
   D {{"#"#"#		
   H 

tQ
'CFFNN""3u5"Ir8   zneeds GPU/CuPy)reasonr  r@   c                    [        5       n[        5       nSnUR                  R                  R	                  US-  5      R                  S5      R                  SS5      nUR                  U5      nUR                  / SQUS-  -  5      nUR                  U5      nUR                  X@US9nUR                  XPUS9n	[        XR                  5       5        g )N   r   r"   r  r@   r   r   r@   r0  r  r&  )r   r   r   randomrandnr
  r   r  r(  r   r	   r   )
r  cupy_ops	numpy_ops
batch_sizer  X_gpur&  lengths_gpur   cols_gpus
             r4   "test_large_seq2col_gpu_against_cpur    s     yH
I J!!*q.188CKKBPQRAq!E !!,*/"BCG$$W-KQw7DkBHD,,.)r8   c                    [        5       n[        5       nSnSU -  S-   nUR                  R                  R	                  X4-  5      R                  S5      R                  SU5      nUR                  U5      nUR                  / SQUS-  -  5      nUR                  U5      nUR                  XPUS9n	UR                  X`US9n
[        XR                  5       5        g )	Nr  r   r@   r"   r  r  r0  r  )r   r   r   r  r  r
  r   r  r(  r'  r	   r   )r  r  r  r  nFd_cols
d_cols_gpur&  r  rs  
d_seqs_gpus              r4   +test_large_backprop_seq2col_gpu_against_cpur    s     yH
I J	
R!B!!*/299)DLLRQST  ##F+J !!,*/"BCG$$W-K''w'GF**:k*RJFNN,-r8   c                    U R                  X!S9nU R                  R                  U5      R                  5       S:  a  g U R                  S/[	        U5      -  SS9nU R                  X#5      nUR                  U:X  d   eUR                  [        U5      UR                  S   4:X  d   eSn[        U5       HE  u  pgU R                  R                  R                  XEXW-    R                  SS9X&   U-  SSS	9  XW-  nMG     g )
Nr   r   r   r   r@   r   r  g{Gz?r  )r   r   rg  r	  r   backprop_reduce_sumr   r   r   r   r   r	   )r   r   r  r&  outstartr   lengths           r4   test_backprop_reduce_sumr    s    
 	A#A
vvzz!}akk1#A,ck2G

!
!!
-C9999Wqwwqz2222Ew'	&&'+++3QTF]TX 	' 	
 		 (r8   c                     U R                  U R                  U5      5      nU H#  nSUR                  5       s=::  a	  S::  a  M    e   e   g )NwJ?gqh ?)softmaxr   r   )r   r  yrh  s       r4   test_softmax_sums_to_oner    sC     	CKKN#A#'')-v----- r8   c                     U R                  U5      nU R                  USS9nU H#  nSUR                  5       s=::  a	  S::  a  M    e   e   g )NTinplacer  grZ|
 ?)r   r  r   )r   r  rh  s      r4   test_softmax_works_inplacer    sL     	AAAt$A#'').w..... r8   r  dYtemperaturereturnc           
      F   SS K n[        U SS9n[        U5      nXB-  nUR                  R                  R	                  USS9nUR                  U5        [        [        [        U5      5      [        [        [        [        UR                  UR                  5      5      5      4$ )Nr   Trequires_gradr  )dim)r3   r   r.   r/   r  backwardr   r   r   Tensorgrad)r  r  r  r3   XtdYtXt_tempYts           r4   torch_softmax_with_temperaturer    s     	!4	(B
2,CG				$	$W"	$	5BKK(2,'(4bgg67*  r8   zneeds PyTorch)r?   rb   rc   c                 z   U R                   R                  SSSSS9R                  SS5      nU R                   R                  SSS9nU R	                  X!S9nU R                  XCUS9n[        X#U5      u  pgU R                   R                  R                  XFSS9  U R                   R                  R                  XWSS9  g )	Nr    r>   r   r   )r  ro  r   )	r   r   r   eyer  backprop_softmaxr  r   r	   )r   r  r  r  Yr  r  dXts           r4   test_softmax_temperaturer    s     	c2s#.66r2>A	Bc	"BA/A			a		=B,QK@GBFFNN""1t"4FFNN""2"6r8   cpu_opsc                    [         R                  " SSS9n[         R                  " SSS9nU[         R                  R                  UR                  S9R                  UR                  5      -  nU[         R                  R                  UR                  S9R                  UR                  5      -  nU R                  X!SS9n[         R                  " X!R                  5      n[        XCSSS	9  [         R                  " S
SS9n[         R                  " SSS9nU[         R                  R                  UR                  S9R                  UR                  5      -  nU[         R                  R                  UR                  S9R                  UR                  5      -  nU R                  X!SS9n[         R                  " UR                  U5      n[        XCSSS	9  U R                  X!SUS9  g )N)r   r   r   r   )r   r   )sizeT)trans2-C6?r  )r   r   )r   r   )trans1)r  r  )r   r   r  uniformr  r   r   gemmdotTr	   )r  Wr  r  rc  s        r4   test_gemm_computes_correctlyr    sr   F#&AF#&A		166		*	2	2177	;;A		166		*	2	2177	;;AQ$'AyyCC HHd6F#&AF#&A		166		*	2	2177	;;A		166		*	2	2177	;;AQ$'Ayya HHd6LLdL*r8   c                     [         R                  " SSS9=p[         R                  " SSS9nU R                  XUS9  [         R                  " U[         R                  " S5      5      (       d   eg )Nr   r   r   )r  )r   r   onesr  array_equal)r  abcs       r4   test_gemm_out_usedr    sW    KKc**A

6%ALL1LQF 34444r8   c           	         U R                  U Vs/ s H  o"PM     sn5      nUR                  S:X  d   eU R                  X1 Vs/ s H  n[        U5      PM     sn5      n[	        X5        U R                  U Vs/ s H  o"PM     snSSS9n[        U5      [        U5      :  d   eU R                  XQ Vs/ s H  n[        U5      PM     snSS9n[	        X5        g s  snf s  snf s  snf s  snf )Nr@   r   )padr   )r  )flattenr   	unflattenr   r	   )r  r  r2   flatunflatflat2unflat2s          r4    test_flatten_unflatten_roundtripr  '  s     ??q>q!Aq>*D99>>ta%8ac!fa%89FAOON1QNO=Eu:D	!!!':1A':BGA *%8*':s   CC 
9C%4C*
c           
         U R                   R                  SSUS9U R                   R                  SSUS9/nU R                   R                  R                  U R	                  U5      / SQ/ SQ/5        U R                   R                  R                  U R	                  USS9/ S	Q/ S
Q/5        U R                   R                  SSUS9R                  SS5      U R                   R                  SSUS9R                  SS5      /nU R                   R                  R                  U R	                  U5      SS/SS/SS/SS//SS/SS/SS/SS///5        U R                   R                  R                  U R	                  USS9SS/SS/SS/SS/SS//SS/SS/SS/SS/SS///5        [        R                  " [        SS9   U R	                  USS9  S S S 5        g ! , (       d  f       g = f)Nr@   r   r   r   )r@   r   r   r   r@   r   r   r   r0  )round_to)r@   r   r   r   r   r   r   r   )r@   r   r   r   r   r   r   r   r   r1  r   r   r-  r/  z'Rounding for padding must at least be 1match)	r   r   r   r	   r  r   r   r   r   )r   r   r  s      r4   test_padr  5  s    
q!5	)366==AU=+KLAFFNN""3771:l/KLFFNN""A!9;S T
 	a%(00A6a%(00A6	A FFNN""
VaVaVaV,VaVaVaV,	
 FFNN""AVaVaVaVaV4VaVaVaVaV4	
 
z)S	TA 
U	T	Ts   >G
G&c           	         U R                  SS/SS/SS/SS/SS//US9nU R                  SS	/5      nU R                  R                  R	                  U R                  X#5      S
S/SS//5        U R                  / SQ5      nU R                  R                  R	                  U R                  X#5      S
S/SS/SS//5        [        R                  " [        5         U R                  X R                  R                  / SQSS95        S S S 5        [        R                  " [        5         U R                  X R                  R                  / SQSS95        S S S 5        g ! , (       d  f       N]= f! , (       d  f       g = f)Nrb   rc   rd   r   r   r   r   r   r         "@r   r   r   r   r   r   r   r   r   r   r  r    r   r   )r  r(  r   r   r	   r*   r   r   r   r   r   r   r   r  r&  s       r4   test_reduce_sumr  V  s<    	
sc3Z#sc3Z#sDE 	 	A mmQF#GFFNN""q"c4[3*$=
 mmI&GFFNN""q"c4[3*sCj$I 
z	"q&&,,|3,?@ 
# 
z	"q&&,,~S,AB 
#	" 
#	" 
#	"s   ,E',E-
E*-
E;c           	         [         R                  " [        SS9   U R                  U R                  R                  SSUS9R                  SS5      U R                  R                  SS/S	S95        S S S 5        g ! , (       d  f       g = f)
Nlengths must ber  r@   r/  r   r   r   r  r$   )r   r   r   r  r   r   r   r   r)  s     r4   )test_backprop_fails_with_incorrect_lengthr  n  sh     
z);	<FFMM!QeM,44Q:FFLL"aL0	
 
=	<	<s   AA99
Bc                    U R                  SS/SS/SS/SS/S	S
//US9nU R                  SS/5      nU R                  X#5      u  pEU R                  R                  R                  XPR                  / SQ5      5        U R                  R                  R                  USS/SS//5        U R                  / SQ5      n[        R                  " [        SS9   U R                  X#5        S S S 5        U R                  / SQ5      n[        R                  " [        SS9   U R                  X#5        S S S 5        g ! , (       d  f       NU= f! , (       d  f       g = f)Nrb   r   rc   r   rd   r   r   r  r   r   r   r   r   r   r   r   r   all sequence lengths must be > 0r  r   r   r@   )lengths must sum up to the number of rows)r  r(  r&   r   r   assert_array_equalr	   r   r   r   r'   r   )r   r   r  r&  r  starts_endss         r4   test_reduce_firstr  x  s    	
sc3Z#sc3Z#tEU 	 	A mmQF#G%%a1NAFFNN%%k==3KLFFNN""1SzC:&>?mmI&G	z)L	M# 
N mmI&G	z)U	V# 
W	V	 
N	M 
W	V   D5E5
E
Ec           	          U R                  SS/SS//US9nU R                  / SQ5      nU R                  X#5      nU R                  R                  R                  USS/SS/SS/SS/SS//5        g )Nrb   rd   rc   r   r   r  r   )r  r(  backprop_reduce_firstr   r   r	   )r   r   r  r  r  s        r4   test_backprop_reduce_firstr    s{     
c
S#J/u	=B--	*K		"	"2	3BFFNN""
c3Z#sc3Z#sc3ZHr8   c                    U R                  SS/SS/SS/SS/S	S
//US9nU R                  SS/5      nU R                  X#5      u  pEU R                  R                  R                  XPR                  SS/5      5        U R                  R                  R                  USS/S	S
//5        U R                  / SQ5      n[        R                  " [        SS9   U R                  X#5        S S S 5        U R                  / SQ5      n[        R                  " [        SS9   U R                  X#5        S S S 5        g ! , (       d  f       NU= f! , (       d  f       g = f)Nrb   r   rc   r   rd   r   r   r  r   r   r   r   r   r   r  r  r  r  r  )r  r(  r'   r   r   r  r	   r   r   r   r   )r   r   r  r&  r  lastss         r4   test_reduce_lastr    s    	
sc3Z#sc3Z#tEU 	 	A mmQF#Gq*HAFFNN%%e]]Aq6-BCFFNN""1SzC;&?@mmI&G	z)L	M# 
N mmI&G	z)U	V# 
W	V	 
N	M 
W	Vr  c           	          U R                  SS/SS//US9nU R                  SS/5      nU R                  X#5      nU R                  R                  R                  USS/SS/SS/SS/SS//5        g )	Nrb   rd   rc   r   r   r   r   r   )r  r(  backprop_reduce_lastr   r   r	   )r   r   r  r  r  s        r4   test_backprop_reduce_lastr    s     
c
S#J/u	=BMM1a&!E		!	!"	,BFFNN""
c3Z#sc3Z#sc3ZHr8   c                 v   U R                   R                  SUS9nX R                   R                  R                  SSUR                  5      -  nU R                   R                  / SQSS9nU R                  X#5      u  pEUR                  U:X  d   eU R                   R                  US:  5      (       d   eU R                   R                  XRR                  S   :  5      (       d   eSn[        U5       HD  u  pxX&Xh-    R                  SS9n	U R                   R                  R                  XG   U	5        Xh-  nMF     g )	N)r-  r   r   r  r@   )r   r   r   r   r   r  )r   r   r  r  r   r   r(   r   r   r   r	  r   r	   )
r   r   r  r&  maxesr  r  r   r  truths
             r4   test_reduce_max_smr    s    	V5)A		r1agg	..Affll9Cl0G>>!-LE;;%66::eqj!!!!66::eggaj())))Ew'	%.)--1-5&&ux7 (r8   c           	         U R                   R                  SUS9nX R                   R                  R                  SSUR                  5      -  nU R                   R                  / SQSS9nU R                  X#5      u  pEUR                  U:X  d   eU R                   R                  US:  5      (       d   eU R                   R                  XRR                  S   :  5      (       d   eSn[        U5       HD  u  pxX&Xh-    R                  SS9n	U R                   R                  R                  XG   U	5        Xh-  nMF     [        R                  " [        5         U R                  X R                   R                  / S	QSS95        S S S 5        [        R                  " [         5         U R                  X R                   R                  / S
QSS95        S S S 5        [        R                  " [         5         U R                  X R                   R                  / SQSS95        S S S 5        g ! , (       d  f       N= f! , (       d  f       Nn= f! , (       d  f       g = f)N)rO  r   r   r  r@   )r   r   r   r-  r   r   r  r  r  )r   r   r   r   r-  )r   r   r  r  r   r   r(   r   r   r   r	  r   r	   r   r   r   r   )
r   r   r   r&  r  r  r  r   r  r  s
             r4   test_reduce_maxr    s    	WE*A		r1agg	..Affll<sl3G >>!-LE;;%66::eqj!!!!66::eggaj())))Ew'	%.)--1-5&&ux7 (
 
z	"q&&,,|3,?@ 
# 
z	"q&&,,~S,AB 
# 
z	"q&&,,c,BC 
#	" 
#	" 
#	" 
#	"s$   ,H%",H60,I%
H36
I
Ic           
         U R                  U R                  R                  SSUS9R                  SS5      U R                  R	                  / SQ/ SQ/5      R                  S5      U R                  R	                  SS/SS95      nUR                  U:X  d   eU R                  R                  R                  U/ S	Q/ S
Q/ SQ/ SQ/ SQ/5        [        R                  " [        5         U R                  U R                  R                  SSSS9R                  SS5      U R                  R	                  / SQ/ SQ/5      R                  S5      U R                  R	                  SS/SS95        S S S 5        [        R                  " [        5         U R                  U R                  R                  SSUS9R                  SS5      U R                  R	                  / SQ/ SQ/5      R                  S5      U R                  R	                  SS/SS95        S S S 5        [        R                  " [        5         U R                  U R                  R                  SSUS9R                  SS5      U R                  R	                  / SQ/ SQ/ SQ/5      R                  S5      U R                  R	                  / SQSS95        S S S 5        g ! , (       d  f       GNa= f! , (       d  f       N= f! , (       d  f       g = f)Nr@   r/  r   r   r   )r   r@   r   )r@   r   r@   r$   r  )r   rc   r   )rb   r   r   )r   r   r   )r   r   r   r   )r   r   r   r  )backprop_reduce_maxr   r   r   r   r
  r   r   r	   r   r   r   r   r  s      r4   test_backprop_reduce_maxr    s@    
	 	 a%(00A6i+,33G<aV7+
B
 88uFFNN""
	
	 
z	"FFMM!QcM*221a8FFLL)Y/077@FFLL!QwL/	
 
# 
z	"FFMM!QeM,44Q:FFLL)Y/077@FFLL"aL0	
 
# 
z	"FFMM!QeM,44Q:FFLL)Y	:;BB7KFFLL'L2	
 
#	" 
#	" 
#	" 
#	"s'   &BKBK4BK(
K
K%(
K6c           	         U R                  SS/SS/SS/SS/SS//US9nU R                  SS	/5      nU R                  R                  R	                  U R                  X#5      SS/SS//5        U R                  / S
Q5      nU R                  R                  R	                  U R                  X#5      SS/SS/SS//5        U R                  SS/SS/SS//US9nU R                  SS/5      nU R                  R                  R	                  U R                  X#5      SS/SS//5        [        R                  " [        5         U R                  X R                  R                  SS/SS95        S S S 5        [        R                  " [        5         U R                  X R                  R                  SS/SS95        S S S 5        g ! , (       d  f       N]= f! , (       d  f       g = f)Nrb   rc   rd   r   r   r   r   r   r   r  r   r   r   r  r   )r  r(  r   r   r	   r)   r   r   r   r   r   r  s       r4   test_reduce_meanr    s    	
sc3Z#sc3Z#sDE 	 	A mmQF#GFFNN""#sCj3*%=
 mmI&GFFNN""#sCj3*sCj%I
 	SzC:Sz:%HAmmQF#GFFNN""#sCj3*%= 
z	"66<<Ac<:; 
# 
z	"66<<Qs<;< 
#	" 
#	" 
#	"s    ,G,G
G
G"c           
      R   U R                  U R                  R                  SSUS9R                  SS5      U R                  R	                  SS/SS95      nUR
                  U:X  d   eU R                  R                  R                  U/ SQ/ SQ/ SQ/ SQ/ S	Q/ S	Q/5        [        R                  " [        S
S9   U R                  U R                  R                  SSUS9R                  SS5      U R                  R	                  SS/SS95        S S S 5        g ! , (       d  f       g = f)Nr@   r/  r   r   r   r   r$   )g      ?r?   g      ?)rc   g      @rd   r  r  r  )backprop_reduce_meanr   r   r   r   r   r   r	   r   r   r   r  s      r4   test_backprop_reduce_meanr  /  s     
	!	!a%(00A6aV7+
B 88uFFNN""
	

 
z);	<  FFMM!QeM,44Q:FFLL"aL0	
 
=	<	<s   9AD
D&	reductionc                 L   [        X5      n[        U SU 35      nU R                  / 5      nU" U R                  SUS9U5      nUS:X  a  Uu  pgU" XgU5      nO*[        U[        5      (       a  Uu  piU" Xi5      nOU" Xe5      nUR
                  S:X  d   eUR
                  S:X  d   eg )N	backprop_)r   r    r   r(   r   r(  r   r`  tupler   
r   r   r  funcbackprop_funcr&  r  r  r  extras
             r4   test_reduce_empty_batchr  J  s     3"DC9YK!89MmmBGSYYweY,g6AL 1W-	Au		1$1&77g88wr8   c                 P   [        X5      n[        U SU 35      nU R                  SS/5      nU" U R                  SUS9U5      nUS:X  a  Uu  pgU" XgU5      nO*[        U[        5      (       a  Uu  piU" Xi5      nOU" Xe5      nUR
                  S:X  d   eUR
                  S:X  d   eg )Nr  r   r   )r   r   r   r(   )r   r   r  r
  s
             r4   test_reduce_empty_hiddenr  a  s     3"DC9YK!89MmmQF#GSYYvUY+W5AL 1W-	Au		1$1&77f88vr8   reduction_raisesc                 p   Uu  p4[        X5      nU R                  SS/SS/SS/SS/SS//US9nU R                  / SQ5      nU(       a,  [        R                  " [
        5         U" Xg5        S S S 5        g U R                  R                  R                  U" Xg5      S	   S
S
/5        g ! , (       d  f       g = f)Nrb   rc   rd   r   r   r   r   r  r@   r   )	r   r  r(  r   r   r   r   r   r	   )r   r   r  reduction_strr   r  r  r&  s           r4   test_reduce_zero_seq_lengthr  x  s     -M+I
sc3Z#sc3Z#sDE 	 	A mmI&G]]:&a! '& 	&&y'<Q'?#sL	 '&s   #	B''
B5c                     U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eg r-   )r   rI   r   r   isnanr   r   r  r  s      r4   	test_mishr    X     	AAA77aggvv||A""$$$$$r8   )backprop_clipped_linearbackprop_dishbackprop_gelubackprop_gelu_approxbackprop_hard_sigmoidbackprop_hard_swishbackprop_hard_swish_mobilenetbackprop_hard_tanhbackprop_mishbackprop_relubackprop_relu_kr  backprop_swishc                    [        X5      n[        R                  " U5      R                  R	                  5        Vs/ s H/  nUR
                  [        R                  R                  :X  d  M-  UPM1     nn[        U5      S:X  a  [        R                  " [        5         U" U R                  R                  SUS9U R                  R                  SUS9U R                  R                  SUS95        S S S 5        [        R                  " [        5         U" U R                  R                  SUS9U R                  R                  SUS9U R                  R                  SUS95        S S S 5        g [        R                  " [        5         U" U R                  R                  SSUS9U R                  R                  SSSUS95        S S S 5        g s  snf ! , (       d  f       N= f! , (       d  f       g = f! , (       d  f       g = f)Nr   r    r   r   r  r  )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   rx   backpropr   positional_argss         r4   .test_eltwise_backprop_rejects_incorrect_shapesr*    s   * sH ""8,77>>@@A99))/// 	
@  
 ?q ]]:&Ru-Qe,Ru- ' ]]:&Ru-Ru-Qe, '& ]]:&c2U3aRu5 '&' '& '& '&s1   ,G,GAGAG$=G5
G!$
G25
Hc                 l   U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eUS:  R                  5       UR                  :X  d   eUS:*  R                  5       UR                  :X  d   eg )Nr   r   )r   ro   r   r   r  r   r   r  r  s      r4   test_relu_kr,    s     	AA

1A77aggvv||A""$$$$F<<>QVV###H>>qvv%%%r8   c                     U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eg r-   )r   rr   r   r   r  r   r  s      r4   
test_swishr.    sX     	AA		!A77aggvv||A""$$$$$r8   c                 l   U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eUS:  R                  5       UR                  :X  d   eUS:*  R                  5       UR                  :X  d   eg )Nr   rb   )r   rp   r   r   r  r   r   r  r  s      r4   test_hard_sigmoidr0    s     	AAA77aggvv||A""$$$$F<<>QVV###H>>qvv%%%r8   c                 l   U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eUS:  R                  5       UR                  :X  d   eUS:*  R                  5       UR                  :X  d   eg )Nr_  rb   )r   rq   r   r   r  r   r   r  r  s      r4   test_hard_tanhr2    s     	AAaA77aggvv||A""$$$$I??&&&H>>qvv%%%r8   c                     U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eg r-   )r   rs   r   r   r  r   r  s      r4   test_hard_swishr4    sY     	AAqA77aggvv||A""$$$$$r8   c                     U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eg r-   )r   rt   r   r   r  r   r  s      r4   test_hard_swish_mobilenetr6    s[     	AA  #A77aggvv||A""$$$$$r8   c                     U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eg r-   )r   rv   r   r   r  r   r  s      r4   test_gelu_approxr8    sY     	AAA77aggvv||A""$$$$$r8   c                     U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eg r-   )r   ru   r   r   r  r   r  s      r4   	test_dishr:    r  r8   c                     U R                  U5      nU R                  U5      nUR                  UR                  :X  d   eU R                  R	                  U5      R                  5       (       a   eg r-   )r   rl   r   r   r  r   r  s      r4   	test_gelur<    r  r8   c                     U R                  U5      nU R                  UR                  5      nU R                  X!5      nUR                  UR                  :X  d   eUS:H  R	                  5       (       d   eg )Nr   )r   r   r   r"  r   )r   r  r   r  s       r4   test_backprop_mishr>  %  s]     	AAIIaggE			5	$B88qww!G==???r8   c                    US:X  aP  US-  U-  US-  -   US-  U-  -   US-  -   n[        SU 5       H"  nXbS-  U-  US-  -   US-  U-  -   US-  -   -  nM$     OZUS-  U-  US-  -   US-  US-  -  -   US-  -   n[        SU 5       H%  nXbS-  U-  US-  -   US-  US-  -  -   US-  -   -  nM'     US-  n[        X$XS:  S9R                  5       nUR                  S5      R                  U:X  d   eU(       aR  U" [        U5      5      n	[        R                  " U4SS9n
U" [        [        U
R                  5       5      U45      5      nO[[        R                  " U4SS	9n	[        R                  " U4SS9n
[        R                  " [        U
R                  5       5      U45      n[        R                  " XX!-  45      n[        R                  " XX!-  45      nXXU
4$ )
Nr@   r   r   )depthbir   r$   )r   r   r   r   )r   r   
initialize	get_paramr  r   r   r  intr   r   )r@  dirsnOr  nIdrawn_paramsrz   lstmr   	size_at_tr  H0C0s                 r4   get_lstm_argsrN  1  s   qyFb=26)BFRK7"q&@q%AQb1f,rAv{:R!VCCH ! Fb=26)BFbAg,>>aGq%AQb1f,rAvq/AABFJJH !Ae	2==?D>>&!&&(222'12 JJj]'B	"C	$8"#=>?XKs3JJj]'B	KK#immo.35	e2:.	/B	e2:.	/By))r8   c           	          U " [        SS5      5      nU " [        SS5      5      nU " [        SS5      5      U-  nU " [        SS5      5      nU " [        SS5      5      n[        XX4XPS9$ )Nr@   r   r   rM  r-  )rH  )r   rN  )rH  r@  rE  rF  r  rG  s         r4   draw_lstm_argsrP  M  sf    !Q EAD	hq"o		%Bhq!n%J	hq"o	BbbDDr8   zdepth,dirs,nO,batch_size,nI))r@   r@   r@   r@   r@   )r@   r@   r   r@   r@   )r@   r@   r   r@   r   )r   r@   r@   r@   r@   )r   r@   r   r   r   )r@   r   r   r@   r@   )r   r   r   r   r   c                     [        5       n[        XX4U5      u  pxpnUR                  XxXU5      nU R                  XxXU5      u  p[        US   US   S   SSS9  [        US   US   S   SSS9  [        XS   SSS9  g Nr   r@   r  r  r  r   )r   rN  lstm_forward_trainingr	   )r   r@  rE  rF  r  rG  reference_opsr   rL  rM  r  rK  	referencer  	fwd_states                  r4   test_lstm_forward_trainingrW  W  s     EM#0bb#Q Fy33FyQI,,V	JLAIaL)A,q/4HIaL)A,q/4HA|$T:r8   zFlaky, skip temporarily)argsc                     Uu  p#pEn[        5       nUR                  X#XEU5      nU R                  X#XEU5      u  p[        U
S   US   S   SSS9  [        U
S   US   S   SSS9  [        XS   SSS9  g rR  )r   rS  r	   )r   rX  r   rL  rM  r  rK  rT  rU  r  rV  s              r4   test_lstm_forward_training_fuzzrZ  n  s    
 $( FyEM33FyQI,,V	JLAIaL)A,q/4HIaL)A,q/4HA|$T:r8   c                     [        [        S5      [        5      (       d   e[        [        S5      [        5      (       d   e SSKJn   [        [        S5      U 5      (       d   e  SSKJn  [        [        S5      U5      (       d   e [        R                  " [        5         [        S5        S S S 5        [        [        5      nUR                  [        :X  d   eg ! [         a#    [        [        S5      [        5      (       d   e Nf = f! [         a#    [        [        S5      [        5      (       d   e Nf = f! , (       d  f       N= f)Nr   cupyr   )AppleOpscpu)BigEndianOpsblah)r`  r   r   r   thinc_apple_opsr]  ImportErrorthinc_bigendian_opsr_  r   r   r   r   r   r   )r]  r_  r   s      r4   test_get_opsrd  |  s    gg&1111gfow////4,'%.(3333
44'%.,7777 
z	" 
#
e*C66U??  4'%.(33334  4'%.(33334	"	"s0   "C &"D #D=*D
	D
*D:9D:=
Ec                  l   [        5       n [        S5         [        5       nUR                  S:X  d   e S S S 5        [        S5         [        5       nUR                  S:X  d   e S S S 5        [        5       nU R                  UR                  :X  d   eg ! , (       d  f       Nf= f! , (       d  f       NF= f)Nr   r\  )r   r   name)	class_opsnew_opss     r4   test_use_opsri    s    !I		!#||w&&& 
 
!#||v%%% 
 G>>W\\))) 
	 
s   BB%
B"%
B3c                     [        S5        [        5       n / SQnU R                  SU5      n[        U5      / SQ/ SQ/:X  d   eU R                  S S 5       U5      n[        U5      / SQSS	/S
//:X  d   e[        U R                  S[        R
                  " U5      5      5      n[        US   [        R                  5      (       d   e[        R                  " US   [        R
                  " / SQ5      5      (       d   e[        R                  " US   [        R
                  " / SQ5      5      (       d   e[        U R                  S S 5       USS95      nU/ SQSS	/S
//:w  d   e[        US   5      S:X  d   e[        US   5      S:X  d   e[        US   5      S:X  d   e[        R                  " [        5         U R                  SS [        S5       5       5        S S S 5        [        R                  " [        5         U R                  SS5        S S S 5        g ! , (       d  f       ND= f! , (       d  f       g = f)Nr   )r@   r   r   r   r   r-  r   )r@   r   r   )r   r   r-  c              3   $   #    U  H  ov   M     g 7fr-   rQ   .0r   s     r4   	<genexpr>!test_minibatch.<locals>.<genexpr>  s     2	1Q	   r  r   r   r-  r@   c              3   $   #    U  H  ov   M     g 7fr-   rQ   rl  s     r4   rn  ro    s     !7Y!Yrp  Tshuffler   r    c              3   $   #    U  H  ov   M     g 7fr-   rQ   rl  s     r4   rn  ro    s     1j1jrp  r   )r   r   	minibatchlistr   r   r`  ra  r  r   r   r   r   r   )r   itemsbatchess      r4   test_minibatchry    s   A

CEmmAu%G=Y	2222mm2	2E:G=YA44443==EMM%$89:Ggaj%--0000WQZy)ABBBBWQZy)ABBBB3==!7Y!7=MNGy1a&1#....wqz?awqz?awqz?a	z	"b1eCj12 
#	z	"b$ 
#	" 
#	"	"	"s   =#HH/
H,/
H=c            	         [        S5        [        5       n [        R                  " / SQ5      n[        R                  " / SQ5      n[	        U R                  SX5      5      n[        R                  " U5      R                  5       SS/SS/SS	/S
S//:X  d   e[	        U R                  SXSS95      n[        U5      S:X  d   e[        US   5      S:X  d   e[        US   5      S:X  d   e[	        U R                  S/ SQ/ SQ5      5      nUSS/SS//SS	/S
S///:X  d   e[        R                  " [        5         U R                  SS [        S5       5       S [        S5       5       5        S S S 5        [        R                  " [        5         U R                  SUS [        S5       5       U5        S S S 5        g ! , (       d  f       NV= f! , (       d  f       g = f)Nr   r  )r   r-  r/  r0  r   r@   r   r-  r   r   r/  r0  Trr  r    c              3   $   #    U  H  ov   M     g 7fr-   rQ   rl  s     r4   rn  "test_multibatch.<locals>.<genexpr>  s     2z!Azrp  r   c              3   $   #    U  H  ov   M     g 7fr-   rQ   rl  s     r4   rn  r|    s     4K
1Q
rp  c              3   $   #    U  H  ov   M     g 7fr-   rQ   rl  s     r4   rn  r|    s     !8Z!Zrp  )r   r   r   r   rv  
multibatchconcatenatetolistr   r   r   r   r   )r   arr1arr2rx  s       r4   test_multibatchr    s   A

C==&D==&D3>>!T01GW%,,.Aq6Aq6Aq6Aq62RRRR3>>!T>>?Gw<1wqz?awqz?a3>>!\<@AGAA'1a&1a&)9::::	z	"r2uSz24Kc
4KL 
#	z	"r4!8U3Z!8$? 
#	" 
#	"	"	"s   :4F>%G>
G
Gc                     [        5       n [        R                  " / SQ[        R                  S9n[	        SS5       H@  n[        U R                  X!5      5      [        SUR                  S   US-
  -
  5      :X  a  M@   e   [        U R                  SU5      5      S:X  d   e[        U R                  UR                  S   S-   U5      5      S:X  d   eg )N)r@   r   r   r   r   r   r@   r    r   r  )	r   r   r   r   r   r   ngramsr	  r   )r   r  ns      r4   test_ngramsr    s    

C===D1b\3::a&'3q$**Q-1q52I+JJJJ szz"d#$)))szz$**Q-!+T23q888r8   z1.9.0zneeds PyTorch 1.9.0
torch_funcirA  )	min_value	max_valuer  )r2   r  c           
      ~   SS K nUu  pg[        X5      n[        U SU-   5      n	U R                  U/US9n
[        U
SS9nU" U5      nU" U
5      nUR	                  5         U
R
                  UR
                  :X  d   eXLd   eU" U
SS9nXL d   eU R                  R                  XSS9(       d   eU R                  R                  XR                  5       S	S9(       d   eU R                  U/US9n
U R                  U/US9nUR                  5       n[        R                  " U	5      nUR                   Vs1 s H  nUS
;   d  M  UiM     nnU1 Sk:X  a  U	" XU
S9nUR
                  U
R
                  :X  d   eUULd   eU	" UXSS9nUUL d   eU R                  R                  UU5      (       d   eU R                  R                  UR                  R                  5       U-  [        U5      SS9(       d   eg USS1:X  a  U	" XS9nUR
                  U
R
                  :X  d   eU R                  R                  UU	" UUSS95      (       d   eU R                  R                  UR                  R                  5       U-  [        U5      SS9(       d   eg USS1:X  a  U	" XS9nUR
                  U
R
                  :X  d   eU R                  R                  UU	" UU
SS95      (       d   eU R                  R                  UR                  R                  5       U-  [        U	" XS95      SS9(       d   eg [!        SU 35      es  snf )Nr   r  r   Tr  r  ro  r   r   )r  r  r  >   r  r  r  )r  r  )r  r  r  r  r  r  )r  )r  r  r  r  r   )r  r  r  z5No PyTorch comparison implemented for parameter set: )r3   r   r   r   r  r   r   isclosedetachcopyr   r   r   r  itemfloatNotImplementedError)r   r   r2   r  r  r3   	func_namepytorch_funcforwardr  x_thincx_torchr  y_thincy_think_inplacedY_thincdY_thinc_inplacesr   r   dx_thincdx_thinc_inplaces                         r4   !test_compare_activations_to_torchr    s8    (Ic%GsK)34H kk1#Uk+Gwd3GWAgGJJL==GMM)))!!!gt4O%%%66>>'>???66>>'88:E>:::kk1#Uk+G{{B4u{-H}}(#A?A.>)>aF?!!H7;~~...x'''#7t
  #3333vv~~h(89999vv~~gll//1B6he~TTT	C;	H0~~...vv~~(GTB
 
 	
 
 vv~~gll//1B6he~TTT	D#;	H0~~...vv~~h"2gtL
 
 	
 
 vv~~LL"$eHX,I&JQV  
 	
 
 "CF8L
 	
= @s   3
L:L:r  r^   c           	         U R                   R                  U/5      nU R                   R                  U R                  USS9U R	                  U5      5      (       d   eU R                   R                  U R                  U R                  S/5      USS9U R                  U R                  S/5      U5      5      (       d   eU R                   R                  U R                  USSS9U R                  U5      5      (       d   eU R                   R                  U R                  U R                  S/5      USSS9U R                  U R                  S/5      U5      5      (       d   eg )Nr   )max_valrb   r>   r?   )slopeoffset)
r   r   r  clipped_linearro   r  	asarray1fr$  rp   r  )r   r2   r  s      r4   test_clipped_linearr    sN    ffnnaS!G66>>#,,Wc,BCJJwDWXXXX66>>##CMM3%$8'3#OCMM3%0':    66>>7#c:C<L<LW<U    66>>##MM3% 'S 	$ 	
 	!!#--"6@	   r8   
byte_order)><=|c                    U R                   R                  U/5      nU R                  X!S9n[        R                  " U R                  U5      U R                  U5      5      (       d   eUS;   a;  UR
                  R                  S5      R                  S5      R                  U:X  d   eg UR
                  R                  UR
                  R                  :X  d   eg )N)r  )r  r  S)r   r   to_numpyr   r  r   newbyteorder	byteorder)r   r  r2   r  s       r4   test_to_numpy_byteorderr  .  s    
 	sAQ.AS\\!_cll1o>>>>Zww##C(55c:DD
RRRww  AGG$5$5555r8   c                  n    [          H  n [        R                  " U 5      nUb  M   e   [        5       c   eg r-   )r   r   get_functionr   )kernel_namecompiled_kernels     r4   test_custom_kernel_compilationr  =  s8    #!..{;*** $ =$$$r8   c                 J    SS/nU[        U R                  USS95      :X  d   eg )NrM  l   4dU5
 r   r   )rv  r   )r   uint64_lists     r4   test_asarray_from_list_uint64r  F  s/     +,K$s{{;h{GHHHHr8   r-   )r   platformtypingr   r   r   r   
hypothesisr   r   hypothesis.strategiesr   r   numpy.testingr	   packaging.versionr
   	thinc.apir   r   r   r   r   r   r   r   thinc.backends._custom_kernelsr   r   r   thinc.compatr   r   r   thinc.typesr   
thinc.utilr   r    r   r   r   MAX_EXAMPLESVANILLA_OPS	NUMPY_OPSBLIS_OPSCPU_OPSXP_OPSappendALL_OPSFLOAT_TYPES	INT_TYPES
REDUCTIONSREDUCE_ZERO_LENGTH_RAISESrw   TORCH_FUNCSmarkparametrizer   filterwarningsr   r   r   r   r   r   r   r   r   
arrays_BOPr  r  r#  r*  rK  rV  r[  r]  rd  rj  rt  rz  r|  r~  skipifr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r*  r,  r.  r0  r2  r4  r6  r8  r:  r<  r>  rN  rP  rW  skiprZ  rd  ri  ry  r  r  floatsr  r  r  r  r  rQ   r8   r4   <module>r     su        & 5 ) %	 	 	 N M ? ?   )  5%jJ	T"k
"

MM')
K=
 )$g	W

 @F &(KK '23 4: (459 6 )9 (" )", '# (# '' (' ' ( (+.((;< = / ) 'Z!!#$ % ( (' )' (. ). (+.	|d3			 ! " 4 / )& (+.
 / )
$ (	|d3			 N ! 4 )N '+.( / ((V '+.+ / (+\ '+.? / (?D '+. / (& '+. / (& '+.; / (; (+.	|d3			 N ! 4 / )N& '+. / (* '+.8 / (8 '+. / (( '+.J / (J6 $-=>1v&* ' ?*( $-=>1v&. ' ?.. (+.	|d3			  ! 4 / )  (	|d3			 . ! 4 ). (	|d3			 / ! 4 )/,1
8X$ 	M/:(8
7 9 ) ;
7 $8g$8x$89+ :+$ $8g$8x$895 :5 G,	|d3			   ! 4 -  (+	"9: ; )> (+.C / )C, (+.
 / )
 (+.$ / )$$ (+. / ) (+.$ / )$$ (+. / ) (+. / )  (+.D / )D8 (+.%
 / )%
P (+.= / )=: (+.
 / )
2 (+.j1 2 / )( (+.j1 2 / )( (+.+-FGM H / )M  (	|d3			 % ! 4 )% '+.$% / ((8 (	|d3			 & ! 4 )& (	|d3			 % ! 4 )% (	|d3			 & ! 4 )& (	|d3			 & ! 4 )& (	|d3			 % ! 4 )% (	|d3			 % ! 4 )% (	|d3			 % ! 4 )% (	|d3			 % ! 4 )% (	|d3			 % ! 4 )% (	|d3			  ! 4 )*8 E E '!; (; 23'	|d3N;  4 ( 4;2	* .@&9 	M/:MGG$44=RS(9i"89{3	|d3#4238
	 4 4 : ) T ;8
v (	|d3		SB	78 9 4 )$ (';<	|d3		SB	786 9 4 = )6 $-=>% ?% (I )Ir8   