
    hN                        S SK r S SKrS SKJrJrJrJrJrJrJ	r	J
r
JrJrJr  S SKrSSKJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,  SSK-J.r.J/r/J0r0  SSK1J2r2  \
" S\S	9r3\
" S
\,S	9r4\Rj                  " S\Rl                  -  5      r7S\Rj                  " S5      -  r8S\Rj                  " S\Rl                  -  5      -  r9 " S S5      r: S\S\S\S\S\!S\\\4   4S jr;S\S\!S\S\4S jr<S\S\=S\=S\=S\=4
S jr>S r?S r@S \S!\S"\S#\S$\S\\\4   4S% jrAS0S& jrBS'\3S\34S( jrCS'\3S\34S) jrDS*\:S\S\4S+ jrES*\:S\S\4S, jrFS-\S.\4S/ jrGg)1    N)AnyIteratorListOptionalSequenceTupleTypeTypeVarUnioncastoverload   )Array1dArray2dArray3dArray4dArrayXd	BatchableDeviceTypesDTypesDTypesFloat	DTypesIntFloats1dFloats2dFloats3dFloats4dFloatsXd	FloatsXdT	GeneratorInts1dInts2dInts3dInts4dIntsXdList2dListXdPaddedShapeSizedGeneratorXp_Floats)get_array_moduleis_xp_arrayto_numpy   )CBlasArrayT)boundFloatsT       @      ?c                      \ rS rSr% Sr\\S'   \r\	\S'    SS\
S\SS	4S
 jjrS\4S jrS	S.S jrSSS.S\\\4   S\S\S\S\4
S jjrSSS.S\\\4   S\S\S\S\S\4S jjrS rS\S\\   4S jrS	S.S\S\S\\   S\4S  jjrS	S.S!\S\S\\   S\4S" jjr   SS#\S$\S%\\   S&\S'\S\4S( jjrS)\S*\S\4S+ jr S)\S,\S-\!S\4S. jr"\#   SS)\$\   S/\\%   S0\S1\S\4
S2 jj5       r&\#   SS)\$\   S/\\%   S0\S1\S\4
S3 jj5       r&\#   SS)\'S/\\%   S0\S1\S\(4
S4 jj5       r&\#   SS)\)S/\\%   S0\S1\S\*4
S5 jj5       r&\#   SS)\+\*   S/\\%   S0\S1\S\*4
S6 jj5       r&   SS)\+\*   S/\\%   S0\S1\S\*4
S7 jjr&\#SS)\S\S0\S\$\   4S8 jj5       r,\#SS)\S\S0\S\$\   4S9 jj5       r,\#SS)\(S\S0\S\'4S: jj5       r,\#SS)\*S\S0\S\)4S; jj5       r,SS)\*S\S0\S\)4S< jjr,\#SS=\$\-   S\.4S> jj5       r/\#SS=\$\   S\04S? jj5       r/ SS=\\$\-   \$\   4   S\14S@ jjr/SA\1S\$\   S\'4SB jr2S=\'S\34SC jr4SA\3S\'4SD jr5SE\6SF\\7   S\84SG jr9SHSISJ.SK\S/\\:   SL\S\!4SM jjr;SHSISJ.SK\SN\S/\\:   SL\S\4
SO jjr<SHSISJ.SK\SN\SP\S/\\:   SL\S\04SQ jjr=SHSISJ.SK\SN\SP\SR\S/\\:   SL\S\>4SS jjr?SHSISJ.SE\6S/\\:   SL\S\84ST jjr@SUSISJ.SK\S/\\A   SL\S\4SV jjrBSUSISJ.SK\SN\S/\\A   SL\S\-4
SW jjrCSUSISJ.SK\SN\SP\S/\\A   SL\S\.4SX jjrDSUSISJ.SK\SN\SP\SR\S/\\A   SL\S\E4SY jjrFSUSISJ.SE\6S/\\A   SL\S\G4SZ jjrHSHSISJ.SE\6S/\\%   SL\S\I4S[ jjrJS\\*SK\S\K4S] jrLS\\*SK\SN\S\(4S^ jrMS\\*SK\SN\SP\S\14
S_ jrNS\\*SK\SN\SP\SR\S\O4S` jrPS\\8SK\S\!4Sa jrQS\\8SK\SN\S\4Sb jrRS\\8SK\SN\SP\S\04
Sc jrSS\\8SK\SN\SP\SR\S\>4Sd jrTS\\8SE\6S\84Se jrUS\\GSK\S\4Sf jrVS\\GSK\SN\S\-4Sg jrWS\\GSK\SN\SP\S\.4
Sh jrXS\\GSK\SN\SP\SR\S\E4Si jrYS\\GSE\6S\G4Sj jrZS\\[SE\6S\[4Sk jr\SHSl.Sm\\>\+\+\+\+\7            4   S/\\%   S\>4Sn jjr]SHSl.Sm\\0\+\+\+\7         4   S/\\%   S\04So jjr^SHSl.Sm\\\+\+\7      4   S/\\%   S\4Sp jjr_SHSl.Sm\\!\+\7   4   S/\\%   S\!4Sq jjr`SHSl.Sm\\8\+\I   4   S/\\%   S\84Sr jjraSUSl.Sm\\\+\   4   S/\\%   S\4Ss jjrbSUSl.Sm\\-\+\+\      4   S/\\%   S\-4St jjrcSUSl.Sm\\.\+\+\+\         4   S/\\%   S\.4Su jjrdSUSl.Sm\\E\+\+\+\+\            4   S/\\%   S\E4Sv jjreSUSl.Sm\\G\+\I   4   S/\\%   S\G4Sw jjrfS	Sl.Sm\\*\+\*   \+\I   4   S/\\%   S\*4Sx jjrgSSm\[S/\\%   S\[4Sy jjrhSSz.S)\iS{\S\i4S| jjrjSSz.S!\iS}\iS{\S\i4S~ jjrkSSz.S}\iS{\S\i4S jjrlSSz.S}\mS{\S\m4S jjrnSSSS.S#\mS{\S\S\7S\m4
S jjroSSS.S\S\S{\S\S\4
S jjrpSSS.S}\mS!\mS\S\7S\m4
S jjrqS!\S}\S\S\4S jrrS\!S\0S\0S)\S\S\s\\s4   4S jrtS\!S\0S\0S)\S\S\4S jruS!\S\S\!S\sS\s\\!4   4
S jrvS)\0S\s\\-4   4S jrwS!\S\-S\S\04S jrxSS)\S{\S\4S jjry SS!\S}\S{\S\4S jjrz     SS)\iS\7S\7S\7S\7S{\S\i4S jjr{     SS!\iS)\iS\7S\7S\7S\7S{\S\i4S jjr|SS)\iS\7S{\S\i4S jjr} SS!\iS)\iS\7S{\S\i4
S jjr~SS)\iS{\S\i4S jjr SS!\iS)\iS{\S\i4S jjrSS)\iS{\S\i4S jjr SS!\iS)\iS{\S\i4S jjrSS)\iS{\S\i4S jjr SS!\iS)\iS}\iS{\S\i4
S jjrSS)\iS{\S\i4S jjr SS!\iS)\iS{\S\i4S jjrSS)\iS{\S\i4S jjr SS!\iS)\iS{\S\i4S jjrSS)\iS{\S\i4S jjr SS!\iS)\iS{\S\i4S jjrS)\iS\i4S jrS)\iS\i4S jrSS)\iS{\S\i4S jjr SS!\iS)\iS{\S\i4S jjrSS)\iS{\S\i4S jjr SS!\iS)\iS{\S\i4S jjr SS)\iS\7S{\S\i4S jjr  SS!\iS)\S\7S{\S\i4
S jjr SS\mS\mS\S\7SS	4
S jjr SS\!S\!S\!S\!S\7S\7S\7S\7S\7S\s\!\!\!\!4   4S jjrS\mS\7S\m4S jrS\mS\mS\74S jrS)\S\S\4S jrS)\S\S\s\\4   4S jrS)\S\S\s\\4   4S jrS)\S\S\4S jrS)\S\S\s\\-4   4S jrS\S\S\4S jrS\S\S\4S jrS\S\S\4S jrS\S\S\4S jrS\S\-S\S\4S jrS\S\S\-4S jrS\S\S\4S jr SS\S\S\S%\\   S\4
S jjrS\S\-S\4S jrS\8S\GS\8S\84S jrS rSrg	)Ops<   basenamexpdevice_type	device_idreturnNc                     Xl         X l        g N)r=   r>   )selfr=   r>   kwargss       L/home/james-whalen/.local/lib/python3.13/site-packages/thinc/backends/ops.py__init__Ops.__init__@   s     '"    c                 H    [        U 5      R                   S3n[        U5      e)zReturn C BLAS function table.z" does not provide C BLAS functions)type__name__NotImplementedError)rB   errs     rD   cblas	Ops.cblasF   s%    d$$%%GH!#&&rG   )
byte_orderc                    [        U[        R                  5      (       a8  U(       a/  UR                  R	                  U5      n[        R
                  " XS9nU$ [        S5      e)Ndtypez,Cannot convert non-numpy from base Ops class)
isinstancenumpyndarrayrR   newbyteorderasarray
ValueError)rB   datarO   rR   s       rD   r.   Ops.to_numpyK   sI    dEMM**

//
;}}T7KKLLrG   Fr/   )shufflebuffersizesequencer[   r\   c                l  ^ ^^^^^ [        TS5      (       d  S[        T5       3n[        U5      eT R                  [	        T5      [        U[        5      (       a  [        R                  " U5      OU5      m[        R                  " [	        T5      5      mUUU UUU4S jn[        U[	        T5      5      $ )aG  Iterate slices from a sequence, optionally shuffled. Slices
may be either views or copies of the underlying data.

The `size` argument may be either an integer, or a sequence of integers.
If a sequence, a new size is drawn before every output.

If shuffle is True, shuffled batches are produced by first generating
an index array, shuffling it, and then using it to slice into the
sequence.

An internal queue of `buffer` items is accumulated before being each
output. Buffering is useful for some devices, to allow the
network to run asynchronously without blocking on every batch.
__len__z-Can't minibatch data. Expected sequence, got c            
   3   0  >#    T(       a  [         R                  R                  T5        / n SnT HS  n[        U5      nU R	                  TR                  TTXU-    5      5        [        U 5      T:  a  U  S h  vN   / n X-  nMU     U  S h  vN   g  N N7fNr   rT   randomr[   intappend
_get_batchlen)	queueir]   r\   indicesrB   r^   r[   sizess	      rD   _iter_items"Ops.minibatch.<locals>._iter_itemsu   s     $$W-EA4yT__Xwqt87LMNu:'$$$E	   % s$   A5B8B9BBBB)hasattrrI   rX   _get_batch_sizesrh   rS   re   	itertoolsrepeatrT   aranger)   )	rB   r]   r^   r[   r\   rL   rm   rk   rl   s	   ` ```  @@rD   	minibatchOps.minibatchT   s    , x++A$x.AQRCS/!%%MZc5J5J9++D1PT
 ,,s8}-
	 	 k3u:66rG   othersc                  ^ ^^^
^^ U4[        U5      -   m[        S T 5       5      (       d<  SR                  T Vs/ s H  n[        U5       PM     sn5      nSU 3n[	        U5      eT R                  [        U5      [        U[        5      (       a  [        R                  " U5      OU5      m[        R                  " [        U5      5      m
UU
U UUU4S jn	[        U	[        T5      5      $ s  snf )ziMinibatch one or more sequences of data, and yield
lists with one batch per sequence. See ops.minibatch.
c              3   :   #    U  H  n[        US 5      v   M     g7f)r`   N)ro   .0seqs     rD   	<genexpr>!Ops.multibatch.<locals>.<genexpr>   s     @is73	**is   z, z/Can't multibatch data. Expected sequences, got c               3   l  >#    T	(       a  [         R                  R                  T5        / n SnT
 Hq  n[        U5      nTXU-    nU R	                  / 5        T H&  nU S   R	                  TR                  XC5      5        M(     [        U 5      T:  a  U  S h  vN   / n X-  nMs     U  S h  vN   g  N N7fNr   r<   rc   )ri   rj   r]   	idx_batchr^   r\   rk   rB   	sequencesr[   rl   s        rD   rm   #Ops.multibatch.<locals>._iter_items   s     $$W-EA4y#AD1	R  )H"I$$T__X%IJ !*u:'$$$E	   % s$   BB4B0B4*B2+B42B4)tuplealljoinrI   rX   rp   rh   rS   re   rq   rr   rT   rs   r)   )rB   r]   r^   r[   r\   rv   r{   valuesrL   rm   rk   r   rl   s   `  ``     @@@rD   
multibatchOps.multibatch   s     K%-/	@i@@@YY)D)349+)DEFCF8LCS/!%%MZc5J5J9++D1PT
 ,,s8}-	 	" k3u:663  Es   C.c                 <  ^ [        T[        5      (       a  U Vs/ s H  nTU   PM
     nnO0[        T[        5      (       a  [        U4S jU 5       5      nOTU   n[        U5      (       a*  U R	                  U R
                  R                  U5      5      nU$ s  snf )Nc              3   .   >#    U  H
  nTU   v   M     g 7frA    )rz   rj   r^   s     rD   r|   !Ops._get_batch.<locals>.<genexpr>   s     818A;s   )rS   listr   r-   	as_contigr;   rW   )rB   r^   rk   rj   subseqs    `   rD   rg   Ops._get_batch   s    h%%+237ahqk7F3F%((888Fg&Fv^^DGGOOF$;<F 4s   Blengthrl   c                 h    / nSnXA:  a(  UR                  [        U5      5        XCS   -  nXA:  a  M(  U$ r   )rf   next)rB   r   rl   outputrj   s        rD   rp   Ops._get_batch_sizes   s:    jMM$u+&OA j rG   )lengthsr{   nWr   c                V   US:X  d   eUS:X  d   eUR                   S   nUR                   S   nU R                  XBS-  S-   U5      nU R                  USU*  SX%5      XbS2SU24'   XSS2U4'   U R                  XS SX%5      USU* 2US-   S24'   U R                  XdUSU-  S-   -  5      $ )zGiven an (M, N) sequence of vectors, return an (M, N*(nW*2+1))
sequence. The new sequence is constructed by concatenating nW preceding
and succeeding vectors onto each column in the sequence, to extract a
window of features.
r/   Nr   r   r<   )shapealloc3f	reshape3f	reshape2f)rB   r{   r   r   BIcolss          rD   seq2colOps.seq2col   s     Qww$IIaLIIaL||AQ
Q/DbS	2r=S#2#XQU#~~c#hBBTrcT268^~~dqAFQJ'788rG   dYc          
      z   US:X  d   eUS:X  d   eUS-  S-   nUR                   S   nUR                   S   U-  nU R                  XV5      nU R                  XXF5      nUSU* === U R                  XS2SU24   SU5      -  sss& XxSS2U4   -  nXrS=== U R                  USU* 2US-   S24   SU5      -  sss& U$ )zThe reverse/backward operation of the `seq2col` function: calculate
the gradient of the original `(M, N)` sequence, as a function of the
gradient of the output `(M, N*(nW*2+1))` sequence.
r/   Nr   r   r<   )r   alloc2fr   r   )	rB   r   r   r   nFr   r   dXdY3ds	            rD   backprop_seq2colOps.backprop_seq2col   s     Qww$!VaZHHQKHHQK2\\!~~bR+
4RCDNN4SbS>2q99
1b5k
34>>$ttR!VX~"6A>>	rG   xyouttrans1trans2c                     U(       a  UR                   nU(       a  UR                   nUc  U R                  R                  X5      $ U R                  R                  XUS9  U$ )zoPerform General Matrix Multiplication (GeMM) and optionally store
the result in the specified output variable.
r   )Tr;   dot)rB   r   r   r   r   r   s         rD   gemmOps.gemm   sM     AA;77;;q$$GGKK#K&JrG   Xrepsc                 8    U R                   R                  X5      $ rA   )r;   tile)rB   r   r   s      rD   r   Ops.tile   s    ww||A$$rG   Wbc                 .    U R                  XSS9nXC-  nU$ )zFApply a weights layer and a bias to some inputs, i.e.
Y = X @ W.T + b
T)r   )r   )rB   r   r   r   Ys        rD   affine
Ops.affine  s"     IIa4I(	rG   rR   padndim_if_emptyc                     g rA   r   rB   r   rR   r   r   s        rD   flattenOps.flatten       	rG   c                     g rA   r   r   s        rD   r   r     r   rG   c                     g rA   r   r   s        rD   r   r     r   rG   c                     g rA   r   r   s        rD   r   r   +  r   rG   c                     g rA   r   r   s        rD   r   r   5  r   rG   c           	         Ub  [        U5      S:X  a  U R                  SU-  U=(       d    SS9$ [        US   5      nUS   R                  nU Vs/ s H  owR                  S:w  d  M  UPM     nn[        U5      S:X  a  U R                  Xb=(       d    SS9$ [        U5      S:  a  / nU HN  nUR                  UR                  U4UR                  SS -   UR                  S95        UR                  U5        MP     UR                  UR                  U4WR                  SS -   UR                  S95        UnUR                  U5      n	Ub  UR                  XS9n	U	$ s  snf )z.Flatten a list of arrays into one large array.Nr   r   frQ   r/   )rh   allocr,   r   r]   re   rf   zerosrR   concatenaterW   )
rB   r   rR   r   r   r;   shape_if_emptyr   paddedresults
             rD   r   r   ?  s=    9A!::d]2%,3:GGad#1)1VVq[Q)q6Q;::nLS:AAs8q=Fbhhv';177hKLa   MM"((C6AGGABK#7qww(GHA"ZZZ4F *s   E'Ec                     g rA   r   rB   r   r   r   s       rD   	unflattenOps.unflattenZ      rG   c                     g rA   r   r   s       rD   r   r   ^  r   rG   c                     g rA   r   r   s       rD   r   r   b  r   rG   c                     g rA   r   r   s       rD   r   r   h  r   rG   c                 :   [        U5      nUS:  a  [        R                  " US:  X#-   S5      nU R                  R	                  U[        R
                  " U5      5      SS nUS:  a  U Vs/ s H  oUUS PM	     nn[        U5      [        U5      :X  d   eU$ s  snf )zThe reverse/backward operation of the `flatten` function: unflatten
a large array into a list of arrays according to the given lengths.
r   Nr<   )r.   rT   wherer;   splitcumsumrh   )rB   r   r   r   unflatas         rD   r   r   l  s    
 7#7kk'A+w}a@Gq%,,w"78"=7'-.v!gvF.6{c'l***	 /s   ,Bseqsc                     g rA   r   rB   r   round_tos      rD   r   Ops.pad}  r   rG   c                     g rA   r   r   s      rD   r   r     r   rG   c                 h   US:  a  [        SU 35      eU(       d  [        S5      e[        [        S U 5       5      5      S:w  a  [        S5      e[        [        S U 5       5      5      S:w  a  [        S5      e[        [        S U 5       5      5      S:w  a  [        S	5      e[        S
 U 5       5      nX3* U-  -  n[        U5      U4US   R                  SS -   n[        [        U R                  XAS   R                  S95      n[        U5       H  u  pgXuUSUR                  S   24'   M     U$ )zPerform padding on a list of arrays so that they each have the same
length, by taking the maximum dimension across each axis. This only
works on non-empty sequences with the same `ndim` and `dtype`.
r/   z.Rounding for padding must at least be 1, was: zCannot pad empty sequencec              3   8   #    U  H  oR                   v   M     g 7frA   )ndimry   s     rD   r|   Ops.pad.<locals>.<genexpr>  s     ,t88t   z)Cannot pad sequences with different ndimsc              3   8   #    U  H  oR                   v   M     g 7frA   rQ   ry   s     rD   r|   r     s     -99r   z*Cannot pad sequences with different dtypesc              3   >   #    U  H  oR                   S S v   M     g7f)r/   N)r   ry   s     rD   r|   r     s     1DS99QR=Ds   z4Cannot pad sequences that differ on other dimensionsc              3   8   #    U  H  n[        U5      v   M     g 7frA   )rh   ry   s     rD   r|   r     s     3ds#c((dr   r   NrQ   )
rX   rh   setmaxr   r   r   r   rR   	enumerate)rB   r   r   max_seq_lenfinal_shaper   rj   arrs           rD   r   r     s1    a<@
K 
 899s,t,,-2HIIs---.!3IJJs1D112a7STT3d33 	|h..4y+.aqr1BBw

;1gmm
(TUoFA(+1n		!n$% & rG   r   c                 |    / n[        U5       H  u  pEUR                  XSU24   5        M     [        [        U5      $ )zThe reverse/backward operation of the `pad` function: transform an
array back into a list of arrays, each with their original length.
N)r   rf   r   r%   )rB   r   r   r   rj   r   s         rD   unpad	Ops.unpad  s>     "7+IAMM&GVG,- ,FF##rG   c                    U(       dL  [        U R                  SSS5      U R                  S5      U R                  S5      U R                  S5      5      $ [        U5      S:X  a  U R	                  US   US   R
                  S   SUS   R
                  S   5      nU R                  S/UR
                  S   -  5      nU R                  UR
                  S   /5      nU R                  S/5      n[        X#XE5      $ [        U5       VVs/ s H  u  pg[        U5      U4PM     nnnUR                  SS9  U V	Vs/ s H  u  pUPM	     n
n	nU V	Vs/ s H  u  pU	PM	     nn	n[        U Vs/ s H  owR
                  S   PM     sn5      n[        U5      nUS   R
                  S   n[        [        U
 Vs/ s H  oaU   PM	     sn5      nU R                  U5      nUR
                  XU4:X  d	   XU45       eU R                  UR                  S5      5      nUR
                  XU4:X  d   e[        U5       Vs/ s H  nSPM     nn[        U5      n[        U5       H5  nU(       a&  UUUS-
     :  a  US-  nU(       a  UUUS-
     :  a  M  UUU'   M7     [!        U5      [!        U5      :X  d   e[        UU R                  U5      U R                  U5      U R                  U
5      5      $ s  snnf s  snn	f s  snn	f s  snf s  snf s  snf )z4Pack a sequence of 2d arrays into a Padded datatype.r   r/   T)reverser/   r   r   )r'   r   alloc1irh   reshape3r   	asarray1ir   sortr   r   r%   r   r   	transposerangesum)rB   r   rY   	size_at_tr   rk   rj   r{   lengths_indicesr   indices_lengths_nSnBnOr   _batch_size_at_t_current_sizets                       rD   list2paddedOps.list2padded  s   Q1%t||AQVW  Y!^==a$q'--*:AtAw}}Q?OPDsTZZ]':;Inndjjm_5GnnaS)G$7<<7@GVQCHa=GT*'67)&A7,;<OyvFO<$/$3))A,$/0Y!W]]1 Fh7h!Wh78xx~yyRRL(622,6(nnS]]956yyRRL(((',Ry1y!Ay18}rA11A(B#B! 11A(B#B".Q  8}$4 5555NN+,NN8$NN8$	
 	
/ H7</ 8 2s$   K%7K+K1'K71K<
3Lc                 Z   UR                   n[        UR                  5      n[        UR                  5      nS/[	        U5      -  nU R                  UR                  S5      5      n[        UR                  S   5       H  nX&S[        XF   5      24   XSU   '   M     [        [        U5      $ )z;Unpack a Padded datatype to a list of 2-dimensional arrays.Nr   r   )rY   r.   rk   r   rh   r   r   r  r   re   r   r%   )rB   r   rY   rk   r   unpaddedrj   s          rD   padded2listOps.padded2list  s    {{6>>*6>>*-1FS\,A~~dnnY78tzz!}%A#'+<S_+<(<#=HQZ  &FH%%rG   r   dropc                    Ub  US::  a  U R                   R                  USS9$ US:  a  U R                  U5      $ U R                   R                  R	                  SSU5      nX2:  SU-
  -  n[        [        U R                  USS95      $ )zCreate a random mask for applying dropout, with a certain percent of
the mask (defined by `drop`) will contain zeros. The neurons at those
positions will be deactivated during training, resulting in a more
robust network and less overfitting.
r   r   rQ   r5           float32)r;   onesalloc_frd   uniformr   r   rW   )rB   r   r  	coinflipsmasks        rD   get_dropout_maskOps.get_dropout_mask  s     <41977<<S<11S[<<&&GGNN**3U;	!cDj1Hdll4ylABBrG   r  TrR   r   d0r   c          	      @    [        [        U R                  U4X#S95      $ Nr  )r   r   r   rB   r   rR   r   s       rD   alloc1fOps.alloc1f  s      Hdjj"ejIJJrG   d1c          	      @    [        [        U R                  X4X4S95      $ r"  )r   r   r   rB   r   r&  rR   r   s        rD   r   Ops.alloc2f  s      Hdjj"jLMMrG   d2c          	      B    [        [        U R                  XU4XES95      $ r"  )r   r   r   rB   r   r&  r*  rR   r   s         rD   r   Ops.alloc3f  s"     Hdjj""UjPQQrG   d3c          	      B    [        [        U R                  XX44XVS95      $ r"  )r   r   r   rB   r   r&  r*  r.  rR   r   s          rD   alloc4fOps.alloc4f  s#     Hdjj"")9jTUUrG   c          	      >    [        [        U R                  XUS95      $ r"  )r   r   r   rB   r   rR   r   s       rD   r  Ops.alloc_f  s     Hdjj5jIJJrG   int32c          	      @    [        [        U R                  U4X#S95      $ r"  )r   r    r   r#  s       rD   r   Ops.alloc1i&  s      FDJJuEJGHHrG   c          	      @    [        [        U R                  X4X4S95      $ r"  )r   r!   r   r(  s        rD   alloc2iOps.alloc2i/  s      FDJJxuJJKKrG   c          	      B    [        [        U R                  XU4XES95      $ r"  )r   r"   r   r,  s         rD   alloc3iOps.alloc3i9  s"     FDJJ|5JNOOrG   c          	      B    [        [        U R                  XX44XVS95      $ r"  )r   r#   r   r0  s          rD   alloc4iOps.alloc4iD  s#     FDJJ'7uJRSSrG   c          	      >    [        [        U R                  XUS95      $ r"  )r   r$   r   r4  s       rD   alloc_iOps.alloc_iP  s     FDJJuJGHHrG   c                    [        U[        5      (       a  U4nU(       a  U R                  R                  XS9$ U R                  R	                  XS9$ )z%Allocate an array of a certain shape.rQ   )rS   re   r;   r   emptyr4  s       rD   r   	Ops.allocY  sE     eS!!HE77===4477===44rG   arrayc                 B    [        [        U R                  X45      5      $ rA   )r   r   reshaperB   rH  r   s      rD   reshape1Ops.reshape1i  s    GT\\%788rG   c                 D    [        [        U R                  XU45      5      $ rA   )r   r   rJ  rB   rH  r   r&  s       rD   reshape2Ops.reshape2l  s    GT\\%b:;;rG   c           	      D    [        [        U R                  XX445      5      $ rA   )r   r   rJ  rB   rH  r   r&  r*  s        rD   r   Ops.reshape3o  s    GT\\%b>??rG   c           
      F    [        [        U R                  XX4U45      5      $ rA   )r   r   rJ  rB   rH  r   r&  r*  r.  s         rD   reshape4Ops.reshape4r  s    GT\\%bb1ABCCrG   c                 B    [        [        U R                  X45      5      $ rA   )r   r   rJ  rK  s      rD   	reshape1fOps.reshape1fu  s    Hdll5%899rG   c                 D    [        [        U R                  XU45      5      $ rA   )r   r   rJ  rO  s       rD   r   Ops.reshape2fx  s    Hdll5r(;<<rG   c           	      D    [        [        U R                  XX445      5      $ rA   )r   r   rJ  rS  s        rD   r   Ops.reshape3f{  s    Hdll5r,?@@rG   c           
      F    [        [        U R                  XX4U45      5      $ rA   )r   r   rJ  rV  s         rD   	reshape4fOps.reshape4f~  s!     Hdll5rr2BCDDrG   c                 $    U R                  X5      $ rA   rJ  rB   rH  r   s      rD   	reshape_fOps.reshape_f      ||E))rG   c                 B    [        [        U R                  X45      5      $ rA   )r   r    rJ  rK  s      rD   	reshape1iOps.reshape1i  s    FDLL677rG   c                 D    [        [        U R                  XU45      5      $ rA   )r   r!   rJ  rO  s       rD   	reshape2iOps.reshape2i  s    FDLLR9::rG   c           	      D    [        [        U R                  XX445      5      $ rA   )r   r"   rJ  rS  s        rD   	reshape3iOps.reshape3i  s    FDLLR=>>rG   c           
      F    [        [        U R                  XX4U45      5      $ rA   )r   r#   rJ  rV  s         rD   	reshape4iOps.reshape4i  s    FDLLRR0@ABBrG   c                 $    U R                  X5      $ rA   rd  re  s      rD   	reshape_iOps.reshape_i  rh  rG   c                 p    [        U[        5      (       a  U4n[        [        UR	                  U5      5      $ )zReshape an array.)rS   re   r   r1   rJ  re  s      rD   rJ  Ops.reshape  s,    eS!!HEFEMM%011rG   rQ   rY   c                <    [        [        U R                  XS95      $ NrQ   )r   r   rW   rB   rY   rR   s      rD   	asarray4fOps.asarray4f       Hdll4l=>>rG   c                <    [        [        U R                  XS95      $ r{  )r   r   rW   r|  s      rD   	asarray3fOps.asarray3f  r  rG   c                <    [        [        U R                  XS95      $ r{  )r   r   rW   r|  s      rD   	asarray2fOps.asarray2f  r  rG   c                <    [        [        U R                  XS95      $ r{  )r   r   rW   r|  s      rD   	asarray1fOps.asarray1f  r  rG   c                <    [        [        U R                  XS95      $ r{  )r   r   rW   r|  s      rD   	asarray_fOps.asarray_f  r  rG   c                <    [        [        U R                  XS95      $ r{  )r   r    rW   r|  s      rD   r   Ops.asarray1i       FDLLL;<<rG   c                <    [        [        U R                  XS95      $ r{  )r   r!   rW   r|  s      rD   	asarray2iOps.asarray2i       FDLLL;<<rG   c                <    [        [        U R                  XS95      $ r{  )r   r"   rW   r|  s      rD   	asarray3iOps.asarray3i  r  rG   c                <    [        [        U R                  XS95      $ r{  )r   r#   rW   r|  s      rD   	asarray4iOps.asarray4i  r  rG   c                <    [        [        U R                  XS95      $ r{  )r   r$   rW   r|  s      rD   	asarray_iOps.asarray_i  r  rG   c                Z   [        XR                  R                  5      (       a0  Uc  U$ UR                  U:X  a  U$ U R                  R	                  XS9$ [        US5      (       a  UR                  5       $ Ub  U R                  R                  XS9$ U R                  R                  U5      $ )z,Ensure a given array is of the correct type.rQ   rT   )rS   r;   rU   rR   rW   ro   rT   rH  r|  s      rD   rW   Ops.asarray  s     dGGOO,,}u$wwt99T7##::<77===3377==&&rG   c                     UR                   S   (       a  USUR                  4;   a  U$ Ub  SU0O0 nU R                  R                  " U40 UD6$ )zAllow the backend to make a contiguous copy of an array.
Implementations of `Ops` do not have to make a copy or make it
contiguous if that would not improve efficiency for the execution engine.
C_CONTIGUOUSNrR   )flagsrR   r;   ascontiguousarray)rB   rY   rR   rC   s       rD   r   Ops.as_contig  sP    
 ::n%%D$**3E*EK%*%6'5!Bww((888rG   inplacer  c                   U(       aC  U R                   R                  USSUS9nU R                   R                  U* US9  US-  nUS-  nU$ U R                   R                  USS5      nSSU R                   R                  U* 5      -   -  $ )N      4      4@r   r5         )r;   clipexprB   r   r  s      rD   sigmoidOps.sigmoid  s    Qt3AGGKKK"HA$JAHQt,A#QB/00rG   r   c                ^    U(       a  U R                  USS9  X!-  nU$ XR                  X#S9-  $ )NTr  )dsigmoidrB   r   r   r  s       rD   backprop_sigmoidOps.backprop_sigmoid  s7     MM!TM*GAHa999rG   c                4    U(       a
  USU-
  -  nU$ USU-
  -  $ )Nr/   r5   r   rB   r   r  s      rD   r  Ops.dsigmoid  s%    QJAHa= rG   c                B    U(       a  US-  nUS-  nUS-  nU$ SUS-  -
  $ )Nr   r  r5   r/   r   r  s      rD   dtanh	Ops.dtanh"  s2    !GAIAHAHq!t8OrG   r5   )r  axistemperaturer  r  c                    US:w  a  X-  nU R                   R                  XSS9nX-
  nU R                   R                  U5      nXwR                  USS9-  nU$ Nr5   T)r  keepdims)r;   r   r  r  )rB   r   r  r  r  maxesshiftednew_xs           rD   softmaxOps.softmax+  s\     #AA48)G$t44rG   )r  r  Xsc                   UR                   S:  a  SUR                    3n[        U5      eU R                  R                  USS5      nU R                  R	                  U5      nU R                  U R                  Xb5      U5      nXg-  nU$ )N   z)Softmax currently only supports 2d. Got: r  r  )r   rK   r;   r  r  backprop_reduce_sum
reduce_sum)rB   r  r   r  r  rL   r  summeds           rD   softmax_sequencesOps.softmax_sequences;  sx     77a<=bggYGC%c**WW\\"eT*B))$//%*I7SrG   )r  r  c                L    US:w  a  X$-  nX-  nXQUR                  USS9-  -  nU$ r  r  )rB   r   r   r  r  r   s         rD   backprop_softmaxOps.backprop_softmaxH  s:     #!BV
"&&dT&222	rG   c                 ^    X!-  nU R                  U R                  XC5      U5      nXBU-  -  nU$ rA   )r  r  )rB   r   r   r   r   sum_dXs         rD   backprop_softmax_sequencesOps.backprop_softmax_sequencesR  s7     V))$//"*FP
&j	rG   paramsH0C0r  c                     UR                   UR                   :X  d   eUR                   S   UR                   S   :X  d   e[        XX4U5      u  pgXg4$ Nr/   )r   lstm_forward_training)rB   r  r  r  r   r  r   	fwd_states           rD   r  Ops.lstm_forward_trainingZ  sP     xx288###xx{bhhqk))),V	J|rG   c                 $    [        XX4U5      u  pgU$ rA   )r  )rB   r  r  r  r   r  r   r
  s           rD   lstm_forward_inferenceOps.lstm_forward_inferenceg  s     %V	BrG   r  c                 $    [        XX45      u  pVXV4$ rA   )backprop_lstm)rB   r   r   r  r  r   d_paramss          rD   r  Ops.backprop_lstmr  s     %R&D|rG   c                 B    UR                  SS9nUR                  SS9U4$ )Nr<   r  )argmaxr   )rB   r   whichs      rD   maxout
Ops.maxoutx  s'    b!uu"u~u$$rG   r  Pc           	         U R                  UR                  S   UR                  S   X1R                  S9n[        UR                  S   5       H0  n[        UR                  S   5       H  nXU4   XEXbXV4   4'   M     M2     U$ )Nr   r/   rQ   )r   r   rR   r  )rB   r   r  r  r   r   os          rD   backprop_maxoutOps.backprop_maxout|  sx    \\"((1+rxx{AXX\Frxx{#A288A;'(*a4aqt$% ( $ 	rG   c                 0    U(       d  XS:  -  $ XS:  -  nU$ rb   r   r  s      rD   reluOps.relu  s!    A;QJAHrG   c                 0    U(       d  XS:  -  $ XS:  -  nU$ rb   r   r  s       rD   backprop_reluOps.backprop_relu  s#     Q<
!e	rG   slopeoffsetmin_valmax_valc                     U(       a"  X-  nX-  nU R                   R                  XXQS9$ X-  U-   nU R                   R                  XtU5      $ )Nr   )r;   r  )rB   r   r  r  r  r  r  r   s           rD   clipped_linearOps.clipped_linear  sM     JAKA77<<G<;;i& ww||C'22rG   c                 V   XT-
  U-  nXd-
  U-  n	U R                   R                  U5      R                  UR                  5      nU R                   R                  S5      R                  UR                  5      n
U R                   R	                  X:  X):  -  X:5      nU(       a  X-  nU$ X-  $ )Nr  )r;   float64astyperR   r   )rB   r   r   r  r  r  r  r  lowhighzeror   s               rD   backprop_clipped_linearOps.backprop_clipped_linear  s     5( E)&--agg6wws#**1773WW]]CG15?HBIwrG   nc                 "    U R                  XUS9$ N)r  r  r  )rB   r   r  r  s       rD   relu_k
Ops.relu_k  s    ""1"AArG   c                 "    U R                  XX4S9$ r  r   )rB   r   r   r  r  s        rD   backprop_relu_kOps.backprop_relu_k  s     ++B1+NNrG   c                 &    U R                  USSUS9$ )N皙?      ?)r  r  r  r  r  s      rD   hard_sigmoidOps.hard_sigmoid  s    ""1CW"MMrG   c                 $    U R                  XSSS9$ )Nr  r  )r  r  r	  rB   r   r   r  s       rD   backprop_hard_sigmoidOps.backprop_hard_sigmoid  s     ++BS+IIrG   c                 &    U R                  USSUS9$ )Nr  r5   )r  r  r  r  r  s      rD   	hard_tanhOps.hard_tanh  s    ""1dC"QQrG   c                 $    U R                  XSSS9$ )Nr  r5   )r  r  r	  r  s       rD   backprop_hard_tanhOps.backprop_hard_tanh  s     ++B4+MMrG   c                 d    U(       a  XR                  U5      -  nU$ XR                  U5      -  nU$ rA   r  rB   r   r  r   s       rD   swish	Ops.swish  s/    a AH,,q/!
rG   c                 Z    X0R                  U5      SU-
  -  -   nU(       a  X-  nU$ X-  nU$ r  r  )rB   r   r   r   r  r   s         rD   backprop_swishOps.backprop_swish  s8     Q1q5))GBIf
rG   c                 d    U(       a  XR                  U5      -  nU$ XR                  U5      -  nU$ rA   )r  r  s       rD   
hard_swishOps.hard_swish  s5    ""1%%AH##A&&
rG   c                 P    US-  S-   nSXBS:  '   SXBS:  '   U(       a  X-  nU$ X-  $ )Ng?r  r5   g      @r   g      r   rB   r   r   r  r   s        rD   backprop_hard_swishOps.backprop_hard_swish  s=     Ws]s7t8HBIwrG   c                 x    U(       a  XR                  US-   5      S-  -  nU$ XR                  US-   5      S-  -  $ )Nr     )r  r  s      rD   hard_swish_mobilenetOps.hard_swish_mobilenet  s?    QU#a''AHKKA&*++rG   c                 V    SUS-  S-   -  nSXBS:  '   SXBS:  '   U(       a  X-  nU$ XA-  $ )NgUUUUUU?r4         @r5   r   g      r   r'  s        rD   backprop_hard_swish_mobilenet!Ops.backprop_hard_swish_mobilenet  sB     C#&s7t8HBIwrG   c                     U R                   R                  U5      nUS-  nU R                   R                  X3S9  X-  nUS-  nUS-  nU(       a  X-  nU$ X-  $ )Nr5   r   r/   r  r;   squaresqrt)rB   r   r  tmps       rD   dishOps.dish  s^    ggnnQs
S"gqs
HAH7NrG   c                     U R                   R                  U5      nUS-   nX R                   R                  U5      -  nSU-  U-  nXuS-  -  nXg-  nUS-  nU(       a  X-  nU$ X-  $ )Nr5   r  g      ?r3  )rB   r   r   r  x_sqx_sq_plus_onederivseconds           rD   backprop_dishOps.backprop_dish  su     ww~~a s
GGLL//q4$$KBI:rG   c                 Z   U R                   R                  U5      nU R                   R                  U5      nSnSnSnSnSnSnSSX-  -   -  n	SXy-  U-   U	-  U-   U	-  U-   U	-  U-   U	-  U R                   R                  U* U-  5      -  -
  n
X*-  nUR	                  UR
                  5      nU$ )Ng~Z O?gi<15ҿgWU?g9LW@g-UB?g{=@?r5   )r;   signabsr  r  rR   )rB   r   rA  a1a2a3a4a5pr  r   r   s               rD   erfOps.erf#  s    ww||AGGKKN3;rv{a'2-2R71<rAQFBFJ
 
 
 hjj!
rG   c                 ~    U R                   R                  USS5      nSU R                   R                  U5      -  S-  $ )Nr  r  r/   r   )r;   r  cosh)rB   r   s     rD   sechsq
Ops.sechsq7  s4    GGLLE4(DGGLLO#))rG   c           
      (   SU R                   R                  [        USU R                   R                  US5      -  -   -  5      -   nUS-  nUR	                  UR
                  5      nU(       a  X-  nU$ U R                   R                  U5      nXC-  nU$ )Nr5   gHm?r  r  )r;   tanhSQRT2PIpowerr  rR   rH  )rB   r   r  r6  r   s        rD   gelu_approxOps.gelu_approx<  s    DGGLLA477==A;N0N,N!OPPs
jj!HAHa AHAHrG   c                 `   [        [        U R                  UR                  5      5      nU R                  R                  US5      nSU R                  R                  SU-  SU-  -   5      -  nUSU-  SU-  -   U R                  SU-  SU-  -   5      -  -  nUS-  nXF-  nU(       a  X-  nU$ X-  $ )Nr  r  giND?gF?gFrVvf?gD?)r   r   r  r   r;   rR  rP  rM  )rB   r   r   r  r   Xp3r6  s          rD   backprop_gelu_approxOps.backprop_gelu_approxH  s     )T\\!''23ggmmAq!DGGLLS8a<!?@@	C(Q,.$++Ohl*3
 
 	
 	s

	HBIwrG   c                 :    [        X5      nU(       a  X-  nU$ X-  $ rA   )gaussian_cdf)rB   r   r  cdfs       rD   geluOps.geluX  s"    4#HAHwrG   c                 X    [        X5      U[        X5      -  -   nU(       a  X-  nU$ X-  $ rA   )rZ  gaussian_pdfr'  s        rD   backprop_geluOps.backprop_gelu`  s3     $"Qd)>%>>HBIwrG   	thresholdc           	          XR                   R                  U R                   R                  SU R                   R                  U5      -   5      5      -  nU R                   R	                  X:  X5      nU(       a  XQS S & U$ U$ Nr5   )r;   rP  logr  r   )rB   r   rb  r  r6  r   s         rD   mishOps.mishj  sa     '',,tww{{3Q+?@AAGGMM!.!1aDHHrG   c                    UR                   UR                   :w  a'  SUR                    SUR                    3n[        U5      e[        U5      nX#:  nX'   nX   n	SUS-   -  n
U
SUR                  SU-  5      -  -  n
XR                  SU-  5      -  n
XR                  U5      SU-  S-   -  -  n
UR                  U5      S-   nX-  nUS-  nXR                  U5      U
-  US-  -  -  nU(       a  UnOUR	                  U5      nXU'   U$ )	N!arrays have incompatible shapes:  and g      @r5   r4   r/        @r   )r   rX   r,   r  copy)rB   r   r   rb  r  msgr;   rk   XsubdYsubomegadeltadXsubr   s                 rD   backprop_mishOps.backprop_mishu  s    88qww5bhhZuQWWINCS/!a -ztcz"rvvcDj)))d
###*!344ts"&&,.5!8<=C''"+CG
rG   emaweightsr  	max_decayc                 B    SU-   SU-   -  nXT:  a  UnUSU-
  X-
  -  -  ng )Nr5   g      $@r/   r   )rB   ru  rv  r  rw  decays         rD   update_averagesOps.update_averages  s6     qTAX&EE	cm,,rG   gradientmom1mom2beta1beta2eps
learn_ratemod_ratec
                     [        X5        [        X5        [        X5        X5-  nXF-  nX2SU-
  -  -  nXBU-  SU-
  -  -  nXX9U R                  R                  U5      -  U-   -  -  -  nXX44$ rd  )_check_compatible_shaper;   r5  )
rB   rv  r|  r}  r~  r  r  r  r  r  s
             rD   adamOps.adam  s     	 2.. 	C%K((8#sU{33 	DGGLL4F)F)L!MNN$,,rG   c                 j    [        U5      nUR                  R                  U5      nXB:  a  XU-  -  nU$ rA   )r,   linalgnorm)rB   r|  rb  r;   	grad_norms        rD   clip_gradientOps.clip_gradient  s6    h'IINN8,	!I--HrG   y_truey_predc                     U R                   R                  US-   5      nX-  SU-
  U R                   R                  SU-
  S-   5      -  -   nU* $ )Ng:0yE>r/   )r;   re  )rB   r  r  log_yplosss        rD   loglossOps.logloss  sJ    Vd]+AJ$''++q6zT>Q2R#RRurG   c                 <   U R                  UR                  S   UR                  S   SS9nSn[        U5       H_  u  pVUS:  a  [        SU 35      eXF-   UR                  S   :  a  [	        S5      eU(       a  XXF-    R                  SS9X5'   XF-  nM[  SX5'   Ma     U$ 	Nr   r/   Fr   'all sequence lengths must be >= 0, got )lengths must sum up to the number of rowsr  r  )r   r   r   rX   
IndexErrorr  rB   r   r   r   startrj   r   s          rD   r  Ops.reduce_sum  s    LLq)1771:ULC"7+IAz #J6(!STT!''!*, !LMM044!4< , rG   c                    UR                   S:X  a!  U R                  SUR                  S   5      U4$ U R                  R	                  US:  5      (       d  [        S5      eU R                  UR                  S   S-   SS9nSUS'   UR                  5       USS & US   UR                  S   :w  a  [        S5      eXS S    U4$ )Nr   r/    all sequence lengths must be > 0Fr  r<   r  )	r]   r   r   r;   r   rX   r   r   r  )rB   r   r   starts_endss       rD   reduce_firstOps.reduce_first  s    <<1<<1771:.77ww{{7Q;''?AAll7==#3a#7ulEA!..*ABr?aggaj(HIISb!"K//rG   c                 :   UR                   S:X  a!  U R                  SUR                  S   5      U4$ U R                  R	                  US:  5      (       d  [        S5      eUR                  5       S-
  nUS   S-   UR                  S   :w  a  [        S5      eX   U4$ )Nr   r/   r  r<   r  )r]   r   r   r;   r   rX   r   r  )rB   r   r   lastss       rD   reduce_lastOps.reduce_last  s    <<1<<1771:.77ww{{7Q;''?AA 1$9q=AGGAJ&HIIxrG   c                 :   U R                  UR                  S   UR                  S   SS9nSn[        U5       H^  u  pVUS:  a  [        SU 35      eXF-   UR                  S   :  a  [	        S5      eU(       a  XXF-    R                  SS9X5'   OSX5'   XF-  nM`     U$ r  )r   r   r   rX   r  meanr  s          rD   reduce_meanOps.reduce_mean  s    LLq)1771:ULC"7+IAz #J6(!STT!''!*, !LMM05515=OE , rG   c                    U R                  UR                  S   UR                  S   UR                  SS9nU R                  UR                  S   UR                  S   SS9nSn[	        U5       Hn  u  pgUS::  a  [        SU 35      eXW-   UR                  S   :  a  [        S5      eU(       a*  XXW-    R                  SS9XF'   XXW-    R                  SS9X6'   XW-  nMp     X44$ )	Nr   r/   Fr  r  &all sequence lengths must be > 0, got r  r  )	r   r   rR   r:  r   rX   r  r  r   )rB   r   r   r   r  r  rj   r   s           rD   
reduce_maxOps.reduce_max  s    LLq)1771:QWWELRW]]1-qwwqzG"7+IA{ #I&!RSS!''!*, !LMMU^4;;;C044!4<OE , xrG   d_firstsr  c                 &   UR                   S:X  a)  U R                  SUR                  S   UR                  SS9$ UR                   S:X  a  [	        S5      eU R                  [        US   5      UR                  S   UR                  SS9nXUS S '   U$ )Nr   r/   Tr  z starts_ends must not have size 1r<   )r]   r   r   rR   rX   re   )rB   r  r  r   s       rD   backprop_reduce_firstOps.backprop_reduce_first	  s     q <<8>>!#4HNNRV<WW"?AA\\B (.."38>>QU  
  (;s	rG   d_lastsr  c                     UR                   S:X  a)  U R                  SUR                  S   UR                  SS9$ U R                  [	        US   5      S-   UR                  S   UR                  SS9nXU'   U$ )Nr   r/   Tr  r<   )r]   r   r   rR   re   )rB   r  r  r   s       rD   backprop_reduce_lastOps.backprop_reduce_last  sv    ::?<<7==#37==PT<UU\\b	NQa 0T  
 5		rG   d_sumsc                     U R                  UR                  5       UR                  S   UR                  SS9nSn[	        U5       H%  u  pVUS:  a  [        SU 35      eX   X4XF-   & XF-  nM'     U$ Nr/   Fr  r   r  r   r  r   rR   r   rX   )rB   r  r   r   r  rj   r   s          rD   r  Ops.backprop_reduce_sum  s~    \\KKM6<<?&,,e  
 "7+IAz #J6(!STT)/Bu~&OE	 ,
 	rG   d_meansc                     U R                  UR                  5       UR                  S   UR                  SS9nSn[	        U5       H(  u  pVUS:  a  [        SU 35      eX   U-  X4XF-   & XF-  nM*     U$ r  r  )rB   r  r   r   r  rj   r   s          rD   backprop_reduce_meanOps.backprop_reduce_mean+  s    \\KKM7==+7==  
 "7+IAz #J6(!STT)0f)<Bu~&OE	 ,
 	rG   d_maxesc                 8   U R                  UR                  5       UR                  S   UR                  S9nSn[	        U5       HR  u  pgUS::  a  [        SU 35      eU R                  R                  XEXW-    X&   R                  S5      X   S5        XW-  nMT     U$ )Nr/   rQ   r   r  )r/   r<   )	r   r  r   rR   r   rX   r;   put_along_axisrJ  )rB   r  r  r   r   r  rj   r   s           rD   backprop_reduce_maxOps.backprop_reduce_max7  s     \\'++-q)9\O"7+IA{ #I&!RSSGG""5>*EH,<,<W,EwzST OE , 	rG   idsseedc           	      z    SSK Jn  U" 5       nU R                  UR                  UR	                  USS9U5      5      $ )zSHash a sequence of 64-bit keys into a table with 4 32-bit keys, using
murmurhash3.
r/   NumpyOpsuint64rQ   )	numpy_opsr  r  hashrW   )rB   r  r  r  r  s        rD   r  Ops.hashF  s=     	(J	~~NN9,,S,A4H
 	
rG   keysc           
      x    SSK Jn  U" 5       nU R                  UR                  XR	                  USS95      5      $ )Nr/   r  r  rQ   )r  r  r   ngramsrW   )rB   r  r  r  r  s        rD   r  
Ops.ngramsQ  s;    'J	~~Q 1 1$h 1 GH
 	
rG   NDperiodc                 ^    SSK Jn  U" 5       nU R                  UR                  XX45      5      $ )Nr/   r  )r  r  r  position_encode)rB   r  r  r  r   r  r  s          rD   r  Ops.position_encodeY  s*     	(J	~~i77fJKKrG   tablerk   c                 $    X   R                  SS9$ )Nr/   r  r  )rB   r  rk   s      rD   
gather_addOps.gather_addb  s    ~!!q!))rG   r   c                 N    U R                   R                  R                  XU5      $ rA   )r;   addat)rB   r  rk   r   s       rD   scatter_addOps.scatter_adde  s     ww{{~~ef55rG   c                     U R                  XS   R                  S9n[        U5       H  u  pEXSUSUR                  S   24'   M     U$ )z8Maybe don't need this? Just a quicky to get Jax working.r   rQ   N)r   rR   r   r   )rB   r   r  r   rj   r   s         rD   insert_intoOps.insert_intoj  sG    EA5bMDA&'1l
l?# "rG   )r>   r=   )cpur<   )NFF)Nr   r   r   )r/   rA   )F)r5   r  r  r5   F)rk  F)r  F)gH.?)r5   )i'  N)rJ   
__module____qualname____firstlineno__r:   str__annotations__rT   r;   r*   r   re   rE   r0   rM   r.   r   r   r   boolr)   rt   r   rg   r   rp   r   r   r    r   r   r   r   r   r   r   r   r   r   r%   r   r&   r   r   r   r!   r"   r   r   r   r   r'   r  r  r(   floatr   r  r   r$  r   r   r   r1  r  r   r   r:  r=  r#   r@  r$   rC  r   r   r   rL  rP  r   r   rW  rZ  r   r   ra  rf  rj  rm  rp  rs  rv  r1   rJ  r}  r  r  r  r  r   r  r  r  r  rW   r   r   r  r  r  r3   r  r  r  r  r  r   r  r  r  r  r  r  r  r  r   r  r
  r  r  r  r  r  r!  r$  r(  r,  r0  r7  r>  rI  rM  rS  rW  r\  r`  rf  rs  rz  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  __static_attributes__r   rG   rD   r7   r7   <   s"   D#BN BD#&#;>#	#'u '
 ,0 M /7CN#/7 /7
 /7 /7 
/7l (7CN#(7 (7 	(7
 (7 (7 
(7T	s 8C=  FJ99!$92:62B9	9* EI #19&1A	2 #'  h	
   
*%h %c %h % X ( x   #'>  	
  
   #'<  	
  
   #'  	
  
   #'  	
  
   #'G  	
  
  #'G  	
  
6 8 f 3 tH~   6 F  T&\   7 V # f  
 7 V # f  7 V # f " V V   X x   CD$v,X67	B$G $d3i $F $(
 (
6 (
T
&& 
&V 
&Ce C8E? Cx C$ (1KK $	K
 K 
K (1NN N
 $N N 
N  (1	R	R 	R 		R $	R 	R 
	R$ (1
V
V 
V 	
V
 
V $
V 
V 

V  (1KK $	K
 K 
K &-II 	"	I
 I 
I &-LL L
 	"L L 
L  &-	P	P 	P 		P 	"	P 	P 
	P$ &-
T
T 
T 	
T
 
T 	"
T 
T 

T  &-II 	"	I
 I 
I #,55 	5
 5 
5 9g 93 97 9<g <3 <C <G <@g @3 @C @S @W @Dg D3 DC DS Dc Dg D:x :S :X :=x =S =c =h =Ax AS Ac As Ax AEE#&E,/E58E>AE	E
*x * *( *8v 83 86 8;v ;3 ;C ;F ;?v ?3 ?C ?S ?V ?Cv C3 CC CS Cc Cf C*v *e * *2V 2E 2f 2 #,	?Hhx%0I'JKKL? 	?
 
? #,	?Hhx'@AAB? 	?
 
? #,	?Hhx778? 	?
 
? #,	?Hhuo-.? 	?
 
? #,	?Hhsm+,? 	?
 
? PW=&(3-/0=<DV<L=	= #*	=FHXc]334= 	=
 
= #*	=FHXhsm%<==>= 	=
 
= #*	=FHXhx}.E%FGGH= 	=
 
= PW=&(3-/0=<DV<L=	= #'	'GXg.=>' 	'
 
',9f 9Xf-= 9 9 8= 
1 
1 
1 
1 ?D:: ):7;:	: 9> !) ! !) ! 49 w D W    	
   
" AFSU%+9=MP	 79s%03GL	'28	  	
   
x			 	 		
 	 	 
	%+5=JO	x!	"% %U8V+;%< %( 6 c h h  (  :?'26	 33 3 	3
 3 3 3 
3(   	
     
(B	 Be BD BY B LQOO )O.3ODHO	O
Ni N$ N9 N <AJJ )J48J	J
R9 Rt R	 R <ANN )N48N	N
y 4 I  JO ).7BF	I    <A		 )	48			,i ,$ ,9 , <A		 )	48			i $ 9  <A )48	$Y 9 (*	 *i *

Y 
 
) 
 <A )48	 i $ 9  <A )48	 FK		',	>B			    	
  
@ JP--%,-14-AF-	-& -- - 	-
 - - - - - - 
x8X5	6-4g % G g w 5 H v ( 0h 0 0E(FBR<S 0X  56AQ;R X  8 H v %&@P:Q  /5	H V  
( 
V 
 

H 
v 
( 
(.9?		
 	
c 	
f 	

 
6 
f 
 NRLLL&)L8@8JL	L* *6 *h *66(.68@6	6
rG   r7   r  c_inith_initr   r   r?   c                    [        U 5      nUR                  u  pgnUR                  u  pUS   n[        [        UR	                  XgUR                  S   US-  4SS95      n[        [        UR	                  XgUR                  S   U4SS95      n[        [        UR	                  XgUR                  S   U4SS95      n[        [
        UR	                  X4SS95      n[        [
        UR	                  X4SS95      n/ nSnU H  nUR                  UUU-   45        UU-  nM      SnUn[        U5       GH  nUR                  S   n
[        U5       GH!  nUUU4   R                  SU45      nUUU4   R                  SU45      n[        U UXU5      u  nn[        U5      u  nnnUUU4==   UR                  UUR                  5      -  ss'   UUU4==   U-  ss'   US:X  a  UO
[        U5       GH}  u  nnUS UU-
   nUS UU-
   nUUUUU24   nUS UR                  S    nUUR                  UUR                  5      -  n[        [        UR                  SUS45      5      n[        US S 2S S 2S4   5      n[        US S 2S S 2S4   5      n[        US S 2S S 2S4   5      n UR!                  US S 2S S 2S4   5      n!UU-  n"U"UU!-  -  n"UR#                  UUU U!45      R                  SSU45      R%                  S	5      R                  SUS-  45      n['        UUU R                  S   -   5      nUR!                  U"5      U -  UUUUU24'   UUUUUU24'   U"UUUUU24'   U"nUUUUU24   nGM     GM$     [        [
        UU   R%                  S
5      R                  U	S45      5      n#US:X  a  UR)                  U#5      n#U#nGM     W#XUU44$ )Nr      r   rQ   r/   r<   r   r  )r   r   r/   r   )r,   r   r   r   r   r   rf   r  rJ  _split_weights_transpose_weightsr   r   reversedr   r  rP  hstackr   minr  )$r  r  r  r   r   r;   depthdirsr	  r  nI
batch_sizeGr   CYt2Ct2rk   r  params_iorig_Xrj   dlayer_paramsWxWhbiasendGt3Gt3_hfhihohcCt3Hs$                                       rD   r  r    s    
&	!BllOEGGEAJXrxxaggaj"q& AxMNAXrxxaggaj" =SxIJAXrxxaggaj" =SxIJA
x:"2#>
?C
x:"2#>
?CGE
uz123  HF5\WWQZtA A,&&2w/CA,&&2w/C%3FArx%P"L(-l;LBDadGrvva&GadGtOG)*agXg5FF
s-C%K(-C%K(1eCi(.CIIaL)rvvc244((Hckk2r1+&>?T!Q']+T!Q']+T!Q']+WWT!Q']+3hrBw IIr2r2./Wb!R[)Yy)Wb"q&\*	  #urxx{23%'WWS\B%6!Qc	/"%(!Qc	/"%(!Qc	/"1eCi(? G R 1Q4>>)4<<aWEF19$$Q'A] ^ qQrG   r   r  c                    [        U5      nUu  pVpxUR                  u  ppUR                  S   nUS   n[        [        UR	                  X4UR
                  S95      n[        [        UR	                  XS-  4UR
                  S95      n[        [        UR	                  UR                  S   4UR
                  S95      nSn/ n[        U	5       He  nUR                  / 5        US:X  a  UOX-  n[        U
5       H6  n[        UUUUU5      u  nn[        U5      nUS   R                  UU45        M8     Mg     Sn/ n[        U	5       He  nUR                  / 5        US:X  a  UOX-  n[        U
5       H6  n[        UUUUU5      u  nn[        U5      nUS   R                  UU45        M8     Mg     / nSnU H  nUR                  UUU-   45        UU-  nM      U/[        U	S-
  5       Vs/ s H6  n[        [        UU   R                  S5      R                  US45      5      PM8     sn-   nU Vs/ s H8  oR	                  UR                  S   UR                  S   4UR
                  S9PM:     nn[        [        U	5      5       GH  n[        [        U R                  XU45      R                  S5      5      nUU   nUU   nU
S:  a  UR                  U5      n[        U
5       GH  nUU   U   S   u  n n!n"UU   U   S   u  n#n$n%US:X  a  US   u  n&n'US S n(U(R!                  5         OUS   u  n&n'USS  n(U( H~  u  n)n*[#        U*U)-
  U'U&-
  5      n+[%        UUU&U&U+-   24   UU&U&U+-    UUUU&U&U+-   24   UUUU&U&U+-   24   UUUU)U)U+-   24   5      u  n,n-UUU)U)U+-   24==   U,U!-  -  ss'   U-UU)U)U+-   & U)n&U*n'M     U#UR&                  U-  -  n#U$UR&                  UUU4   -  -  n$U%UR)                  SS9-  n%UUU -  -  nGM     Un GM     WR                  S   WR                  S   :X  d   e/ n.U H)  nU H   u  n/n0U.R                  [+        U/5      5        M"     M+     UUR-                  U.5      4$ s  snf s  snf )	Nr/   r   rQ   r  r<   r   r   r  )r,   r   r   r   r   rR   r   r  rf   r  r  r   rJ  r  r   r  r   r  backprop_lstm_gatesr   r  _untranspose_unsplit_weightsr   )1r   r   r  r  r;   r   r  r  r   r  r  r  r	  r  r  dCdGr  r  all_layer_paramsrj   n_inputsr   r  all_layer_gradslayer_gradsrk   r  r  dXsr   r   r  r  r  dWxdWhd_biasstart_t3end_t3layer_indicesstart_t2end_t2r]   dGt3dCt2
grad_parts	dir_gradsr
  s1                                                    rD   r  r    s   	&	!B JA!E	
BJ	h!8	9B	h!!VAGG<	=BHbhhQ'9hNOH HTV5\#a2bitA%3FAr8X%V"L(-l;LR ''x(@A   HSUO5\r"a2bitA$28QHh$W!K,[9KB&&X'>?   GE
uz123  uqy!!A 	Xqt~~i088!RAB! 
B GI
Ib88QWWQZ,AGG8<bC
IeEl#Hbjj!27AA)LMVqE19''-DtA+A.q1!4LBD.q1!4Q7CfAv#*2; & '%%'#*1: & '$1 &6H,fx.?@0Hx$667x(T/2aHx$667aHx$667aHx$667
d Q8d?223tby@3158ho.# %2  244!8OC244!AqD'>!Cbff!fn$F"r'MBA B O $P 88A;!''!*$$$J&'LIq:9EF ( ' r~~j)))g Js   )=Q0/?Q5rj   r	  r  r  c                 2   SU-  U-  nSU-  nSU-  U-  nSU-  nXXE-    R                  SU-  U45      n	XE-  nXXF-    R                  SU-  45      n
XF-  nXXG-    R                  SU-  U45      nXG-  nXXH-    R                  SU-  45      nXH-  nX4X44U4$ )Nr  rd  )r  rj   r	  r  r  Wx_sizebx_sizeWh_sizebh_sizer  bxr  bhs                rD   r  r  E  s    "frkG"fG"frkG"fG	8-	.	6	6B|	DBH	8-	.	6	6By	ABH	8-	.	6	6B|	DBH	8-	.	6	6By	ABHHrh))rG   c                 |   U u  u  pu  p4[        U5      nUR                  SSUR                  S   45      nUR                  S5      R                  SUR                  S   45      nUR                  S5      R                  S5      R                  S5      nUR                  SSUR                  S   45      nUR                  S5      R                  SUR                  S   45      nUR                  S5      R                  S5      R                  S5      nUR                  nU" U5      nU" U5      nU" U5      U-   nXU4$ )Nr  r<   r   )r  r<   r/   r   r<   )r,   rJ  r   r   r  )r  r  r)  r  r*  r;   ascontigr  s           rD   r  r  U  s     HRhr	"	B	QBHHRL)	*B	i	 	(	("bhhrl);	<B	G		&	&v	.	6	6u	=B	QBHHRL)	*B	i	 	(	("bhhrl);	<B	G		&	&v	.	6	6u	=B##H	"B	"BB<"D4<rG   c                     U u  pn[        U5      nUR                  S   nUR                  S   nUR                  SSU45      R                  S5      R                  SU45      nUR                  SSU45      R                  S5      R                  SU45      nUR                  S5      R                  S5      R                  S5      nUR	                  UR                  SS	9nUR                  UR                  5       X2R                  5       U45      $ )
Nr/   r<   r  r   )r<   r  r,  r-  r   rQ   )r,   r   rJ  r   r   r   ravel)r  r  r  r  r;   r	  r  r   s           rD   r  r  g  s    LBD	"	B	!B	!B	RBK	 	*	*9	5	=	=r2h	GB	RBK	 	*	*9	5	=	=r2h	GB<< **62::5ADHHTZZsH+E>>288:tXXZ?@@rG   dYt3dCt3r  r  r  c                    [        U 5      nUR                  USSS9u  pgpUR                  S   UR                  S   s=:X  a(  UR                  S   s=:X  a  U	R                  S   :X  d   e   eUR                  S   U R                  S   s=:X  a<  UR                  S   s=:X  a(  UR                  S   s=:X  a  UR                  S   :X  d   e   eUR                  U5      n
X
-  nX-  nX[	        U
5      -  -  nX-  nX-  nX-  nX-  nU[	        U	5      -  nU[        U5      -  nU[        U5      -  nU[        U5      -  nUR                  UUUU4SS9nUU4$ )Nr  r<   r  r   )r,   r   r   rP  r  r  r   )r1  r2  r  r  r  r;   r  r	  r
  r  tanhCt3d_ho	d_tanhCt3d_hid_hcd_hfr!  d_At3_hcd_At3_hod_At3_hid_At3_hfdAt3s                         rD   r  r  s  sU    
$	BXXc12X.NBB88A;"((1+C!CCCCCC88A;$**Q-X4::a=XCIIaLXCIIVWLXXXXXggclG>D	Ig&&D9D9D:D9DeBiHhrl"Hhrl"Hhrl"H>>8XxB>LD:rG   c                 n    [        U 5      nUR                  U SS5      n SSUR                  U * 5      -   -  $ )Nr  r  r5   )r,   r  r  )r   r   r;   s      rD   r  r    s:    	!	B 	5$A#r
"##rG   r   c                     U SU -
  -  $ rd  r   r   s    rD   r  r    s    a=rG   c                     SU S-  -
  $ )Nr/   r   r   rA  s    rD   r  r    s    q!t8OrG   opsc                 >    SSU R                  [        U-  5      -   -  $ )z6Gaussian CDF for distribution with mean 0 and stdev 1.r  r5   )rI  	INV_SQRT2rC  r   s     rD   rZ  rZ    s    #	A..//rG   c                 R    [         U R                  R                  SU-  U-  5      -  $ )z6Gaussian PDF for distribution with mean 0 and stdev 1.g      )INV_SQRT_2PIr;   r  rF  s     rD   r_  r_    s"    #&&**TAX\222rG   uvc                     U R                   UR                   :w  a'  SU R                    SUR                    3n[        U5      eg )Nri  rj  )r   rX   )rI  rJ  rm  s      rD   r  r    s;    ww!''1!''%yIo rG   rA   )Hrq   mathtypingr   r   r   r   r   r   r	   r
   r   r   r   rT   typesr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   utilr,   r-   r.   rM   r0   r1   r3   r5  pirQ  rE  rH  r7   r  r  re   r  r  r  r  r  r  r  rZ  r_  r  r   rG   rD   <module>rQ     s               > ; : 		)
)7
+
))C$''M
"$))C. 	TYYsTWW}--s sl)#LD D &D 08D =ED PVD 
8U?D Nc*h c* c* c*e c*L*8 * * *# * * $	A
")18@GO
8X8$ 6 V  0c 0i 0I 0
3c 3i 3I 3
x H rG   