h SSKJr \(dS\;aSSKJr OSSKrSSKrSr Sr Sr S r "S S \5r"S S \5r\R$"\5 "SS\5r\R("\5 "SS\5r\R,"\5 "SS\5r\R0"\5 SrSr"SS\5r\R8"\5 "SS\5r\R<"\5 SrSSK r SSK!r!SSKrSSK"r"SSK#r$SSK%J&r& SSK%J'r' Sr(Sr)\RTr+\RTr,\-"\S\R\5 \-"\S\R\5 \R^\l0\Rb\l2S H r3\)"\\35 M \("\5 \("\5 \("\5 \r4\r5SS!K6J7r7 \"\"RpRs\:"\$RvRyS"55S#55 "S$S%\5r=g!\ a SSKrGNf=f)&) version_info.)_sentencepieceNcSURR5-nSURR <SURR <SU<S3$![Ra SnNOf=f)Nz proxy of )this__repr__ __builtin__ Exception __class__ __module____name__)selfstrthiss P/home/james-whalen/.local/lib/python3.13/site-packages/sentencepiece/__init__.py _swig_reprrs_ 2 2 44"^^668O8OQX ZZ  sAA/.A/c^U4SjnU$)Nc>US:Xa T"XU5 gUS:XaURRU5 g[X5(a2[[ [ U5U5[ 5(a T"XU5 g[SU-5e)Nr thisownz(You cannot add instance attributes to %s)r ownhasattr isinstancegetattrtypepropertyAttributeError)rnamevaluesets rset_instance_attrE_swig_setattr_nondynamic_instance_variable..set_instance_attrsi 6> E " Y  IIMM% T ZT D0I8%T%T E " !Kd!RS S)r!r"s` r*_swig_setattr_nondynamic_instance_variabler&sT r$c^U4SjnU$)Nc>[X5(a([[X5[5(d T"XU5 g[ SU-5e)Nz%You cannot add class attributes to %s)rrrrr)clsrr r!s rset_class_attr?_swig_setattr_nondynamic_class_variable..set_class_attr)s: 3  j1CX&N&N 5 ! !H3!NO Or$r%)r!r*s` r'_swig_setattr_nondynamic_class_variabler,(sP r$c^U4SjnU$)zlClass decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclasscp>T"URURURR55$N)r __bases____dict__copy)r) metaclasss rwrapper$_swig_add_metaclass..wrapper3s's}}cll6G6G6IJJr$r%)r3r4s` r_swig_add_metaclassr61sK Nr$c<\rSrSrSr\"\R5rSrg)_SwigNonDynamicMeta8zKMeta class to enforce nondynamic attributes (no new attributes) for a classr%N) rr __qualname____firstlineno____doc__r,r __setattr____static_attributes__r%r$rr8r88sU9$:J:JKKr$r8c\rSrSr\"SSSS9r\rSr\ Rr Sr Sr S rS rS rS rS r\"\ 5r\"\5r\"\ 5r\"\5r\"\5r\"\5r\"\5rSrSrSr\rSrg)1ImmutableSentencePieceText_ImmutableSentencePiece=c6URR5$r/r rxs r:ImmutableSentencePieceText_ImmutableSentencePiece.>r$c8URRU5$r/rCrEvs rrFrG>AFFJJqMr$The membership flagdoccX[R"U[R"55 gr/)r:ImmutableSentencePieceText_ImmutableSentencePiece_swiginit5new_ImmutableSentencePieceText_ImmutableSentencePiecers r__init__:ImmutableSentencePieceText_ImmutableSentencePiece.__init__As,QQRVXfY]Y]Y_ `r$c.[R"U5$r/)r8ImmutableSentencePieceText_ImmutableSentencePiece__piecerSs r_piece8ImmutableSentencePieceText_ImmutableSentencePiece._pieceEVVW[\\r$c.[R"U5$r/)r:ImmutableSentencePieceText_ImmutableSentencePiece__surfacerSs r_surface:ImmutableSentencePieceText_ImmutableSentencePiece._surfaceHsXXY]^^r$c.[R"U5$r/)r5ImmutableSentencePieceText_ImmutableSentencePiece__idrSs r_id5ImmutableSentencePieceText_ImmutableSentencePiece._idKsSSTXYYr$c.[R"U5$r/)r8ImmutableSentencePieceText_ImmutableSentencePiece__beginrSs r_begin8ImmutableSentencePieceText_ImmutableSentencePiece._beginNrZr$c.[R"U5$r/)r6ImmutableSentencePieceText_ImmutableSentencePiece__endrSs r_end6ImmutableSentencePieceText_ImmutableSentencePiece._endQsTTUYZZr$c.[R"U5$r/)rCImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytesrSs r_surface_as_bytesCImmutableSentencePieceText_ImmutableSentencePiece._surface_as_bytesTsaabfggr$c.[R"U5$r/)rAImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytesrSs r_piece_as_bytesAImmutableSentencePieceText_ImmutableSentencePiece._piece_as_bytesWs__`deer$cSRURURURURUR 5$)Nz3piece: "{}" id: {} surface: "{}" begin: {} end: {} )formatpieceidsurfacebeginendrSs r__str__9ImmutableSentencePieceText_ImmutableSentencePiece.__str__bs7"6$**dggt||"&**dhh8 8r$c4URUR:H=(ay URUR:H=(aY URUR:H=(a9 URUR:H=(a URUR:H$r/)rurvrwrxryrothers r__eq__8ImmutableSentencePieceText_ImmutableSentencePiece.__eq__js ZZ5;; &Y477ehh+>Y4<r%r$rr@r@=s-/IOdeGH`%^^]_Z][hf V Eo.Nx G 12 #B V E 4.C8YHr$r@c\rSrSr\"SSSS9r\rSr\ Rr Sr Sr S rS rS rS r\"\5r\"\5r\"\5r"S S5r\S5rSrSrSr\rSrg)ImmutableSentencePieceTextuc6URR5$r/rCrDs rrF#ImmutableSentencePieceText.vrHr$c8URRU5$r/rCrJs rrFrvrLr$rMrNcX[R"U[R"55 gr/)r#ImmutableSentencePieceText_swiginitnew_ImmutableSentencePieceTextrSs rrT#ImmutableSentencePieceText.__init__ys::4AnAnApqr$c.[R"U5$r/)r'ImmutableSentencePieceText__pieces_sizerSs r _pieces_size'ImmutableSentencePieceText._pieces_size}sEEdKKr$c.[R"X5$r/)r"ImmutableSentencePieceText__piecesrindexs r_pieces"ImmutableSentencePieceText._piecess@@MMr$c.[R"U5$r/)r ImmutableSentencePieceText__textrSs r_text ImmutableSentencePieceText._texts>>tDDr$c.[R"U5$r/)r!ImmutableSentencePieceText__scorerSs r_score!ImmutableSentencePieceText._scores??EEr$c.[R"U5$r/)r,ImmutableSentencePieceText_SerializeAsStringrSs rSerializeAsString,ImmutableSentencePieceText.SerializeAsStringJJ4PPr$c.[R"U5$r/)r)ImmutableSentencePieceText__text_as_bytesrSs r_text_as_bytes)ImmutableSentencePieceText._text_as_bytessGGMMr$c0\rSrSrSrSrSrSr\rSr g)9ImmutableSentencePieceText.ImmutableSentencePieceIteratorcNXlURR5Ulgr/)protorlenrrs rrTBImmutableSentencePieceText.ImmutableSentencePieceIterator.__init__ ::**,r$cUR$r/rrSs r__len__AImmutableSentencePieceText.ImmutableSentencePieceIterator.__len__ xxr$c[U[5(ab[UR5Vs/sHo RR U5PM snUR URUR2$US:aXR-nUS:dXR:a [S5eURR U5$s snf)Nrzpiece index is out of range) rslicerangerrrstartstopstep IndexErrorrris r __getitem__EImmutableSentencePieceText.ImmutableSentencePieceIterator.__getitem__s eU # #16txxAA**$$Q'A%++ejjY^YcYcBcd d 19(("% 19)89 9zz!!%(( B$Cc SRUVs/sHnSR[U55PM sn5$s snf)N pieces {{ {}}}joinrtrrrEs rrzAImmutableSentencePieceText.ImmutableSentencePieceIterator.__str__3yyDIDq+223q6:DIJJI$;rrN rrr:r;rTrrrzr r>r%r$rImmutableSentencePieceIteratorr-)Khr$rc,[RU5$r/)rrrSs rpieces!ImmutableSentencePieceText.piecess ' F Ft LLr$cDUR5UR5:H$r/rr}s rr!ImmutableSentencePieceText.__eq__  # # %)@)@)B BBr$c4[UR55$r/rrrSs rr#ImmutableSentencePieceText.__hash__ $((* ++r$cSRURURSRURVs/sHnSR[ U55PM sn55$s snf)Nztext: "{}" score: {} {}rr)rttextscorerrrrs rrz"ImmutableSentencePieceText.__str__s^F499djj99PTP[P[%\P[1&7&>&>s1v&FP[%\]__&]s$A*r%N)rrr:r;rrrr rTr!delete_ImmutableSentencePieceTextrrrrrrrr text_as_bytesrrrrrrzr>r%r$rrrus-/IOdeGHr%GGLNEFQN E?D^,M V E,MMC,_ Hr$rc\rSrSr\"SSSS9r\rSr\ Rr Sr Sr S r"S S 5r\S 5rS rSrSr\rSrg)ImmutableNBestSentencePieceTextc6URR5$r/rCrDs rrF(ImmutableNBestSentencePieceText.rHr$c8URRU5$r/rCrJs rrFrrLr$rMrNcX[R"U[R"55 gr/)r(ImmutableNBestSentencePieceText_swiginit#new_ImmutableNBestSentencePieceTextrSs rrT(ImmutableNBestSentencePieceText.__init__s??nFxFxFz{r$c.[R"U5$r/)r,ImmutableNBestSentencePieceText__nbests_sizerSs r _nbests_size,ImmutableNBestSentencePieceText._nbests_sizerr$c.[R"X5$r/)r'ImmutableNBestSentencePieceText__nbestsrs r_nbests'ImmutableNBestSentencePieceText._nbestssEEdRRr$c.[R"U5$r/)r1ImmutableNBestSentencePieceText_SerializeAsStringrSs rr1ImmutableNBestSentencePieceText.SerializeAsStringsOOPTUUr$c0\rSrSrSrSrSrSr\rSr g)BImmutableNBestSentencePieceText.ImmutableSentencePieceTextIteratorcNXlURR5Ulgr/)rrrrs rrTKImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__init__rr$cUR$r/rrSs rrJImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__len__rr$c[U[5(ab[UR5Vs/sHo RR U5PM snUR URUR2$US:aXR-nUS:dXR:a [S5eURR U5$s snf)Nrznbests index is out of range) rrrrrrrrrrrs rrNImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__getitem__s eU # #16txxAA**$$Q'A%++ejjY^YcYcBcd d 19(("% 19)9: :zz!!%(( Brc SRUVs/sHnSR[U55PM sn5$s snfNrznbests {{ {}}}rrs rrzJImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__str__rrrNrr%r$r"ImmutableSentencePieceTextIteratorrrr$rc,[RU5$r/)rrrSs rnbests&ImmutableNBestSentencePieceText.nbestss , O OPT UUr$cDUR5UR5:H$r/rr}s rr&ImmutableNBestSentencePieceText.__eq__rr$c4[UR55$r/rrSs rr(ImmutableNBestSentencePieceText.__hash__rr$c SRURVs/sHnSR[U55PM sn5$s snfr)rr rtrrs rrz'ImmutableNBestSentencePieceText.__str__s7 YY$++N+Q)00Q8+N OONs$Ar%N)rrr:r;rrrr rTr&delete_ImmutableNBestSentencePieceTextrrrrrr rrrzr>r%r$rrrst-/IOdeGH|%LLQSV,VVC,PHr$rc .\rSrSr\"SSSS9r\rSr\ Rr Sr Sr S rS rS rS rS rSrSrSrSrSrSrSrSrSrSrSrSrSrSr Sr!Sr"Sr#Sr$S r%S!r&S"r'S#r(S$r)S%r*S&r+S'r,S(r-S)r.S*r/S+r0S,r1S-r2S.r3S/r4S0r5S1r6S2r7S3r8S4r9S5r:S6r;S7rS:r?S;r@S<rAS=rBS>rCS?S?\DS@S@S@S@S@SASBSA4 SCjrESiSDjrFSErGSFrHSGrISHrJSjSIjrKSjSJjrLSjSKjrMSjSLjrNSkSMjrOSlSNjrPSlSOjrQSlSPjrRSlSQjrSSiSRjrTSjSSjrUSjSTjrVSjSUjrWSjSVjrX\YS?4SWjrZ\Y4SXjr[\Y4SYjr\SmSZjr]SmS[jr^SnS\jr_SnS]jr`SlS^jrSlS_jraS`rbSarcSbrdScreSdrfSergSfrhSjSgjriShrjg?)oSentencePieceProcessorc6URR5$r/rCrDs rrFSentencePieceProcessor.rHr$c8URRU5$r/rCrJs rrFrrLr$rMrNcX[R"U[R"55 gr/)rSentencePieceProcessor_swiginitnew_SentencePieceProcessorrSs rrTSentencePieceProcessor.__init__s66t^=f=f=hir$c.[R"X5$r/)r.SentencePieceProcessor_LoadFromSerializedProtor serializeds rLoadFromSerializedProto.SentencePieceProcessor.LoadFromSerializedProtosLLT^^r$c.[R"X5$r/)r,SentencePieceProcessor_SetEncodeExtraOptionsr extra_options rSetEncodeExtraOptions,SentencePieceProcessor.SetEncodeExtraOptionsJJ4^^r$c.[R"X5$r/)r,SentencePieceProcessor_SetDecodeExtraOptionsr%s rSetDecodeExtraOptions,SentencePieceProcessor.SetDecodeExtraOptionsr)r$c.[R"X5$r/)r$SentencePieceProcessor_SetVocabulary)r valid_vocabs r SetVocabulary$SentencePieceProcessor.SetVocabularysBB4UUr$c.[R"U5$r/)r&SentencePieceProcessor_ResetVocabularyrSs rResetVocabulary&SentencePieceProcessor.ResetVocabulary sDDTJJr$c0[R"XU5$r/)r%SentencePieceProcessor_LoadVocabulary)rfilename thresholds rLoadVocabulary%SentencePieceProcessor.LoadVocabularysCCDT]^^r$c0[R"U/UQ76$r/)r'SentencePieceProcessor_CalculateEntropyrargss rCalculateEntropy'SentencePieceProcessor.CalculateEntropysEEdRTRRr$c.[R"U5$r/)r#SentencePieceProcessor_GetPieceSizerSs r GetPieceSize#SentencePieceProcessor.GetPieceSizesAA$GGr$c.[R"X5$r/)r SentencePieceProcessor_PieceToIdrrus r PieceToId SentencePieceProcessor.PieceToIds>>tKKr$c.[R"X5$r/)r SentencePieceProcessor_IdToPiecerrvs r IdToPiece SentencePieceProcessor.IdToPiece>>tHHr$c.[R"X5$r/)rSentencePieceProcessor_GetScorerNs rGetScoreSentencePieceProcessor.GetScore==dGGr$c.[R"X5$r/)r SentencePieceProcessor_IsUnknownrNs r IsUnknown SentencePieceProcessor.IsUnknown rQr$c.[R"X5$r/)r SentencePieceProcessor_IsControlrNs r IsControl SentencePieceProcessor.IsControl#rQr$c.[R"X5$r/)rSentencePieceProcessor_IsUnusedrNs rIsUnusedSentencePieceProcessor.IsUnused&rVr$c.[R"X5$r/)rSentencePieceProcessor_IsByterNs rIsByteSentencePieceProcessor.IsByte)s;;DEEr$c.[R"U5$r/)rSentencePieceProcessor_unk_idrSs runk_idSentencePieceProcessor.unk_id,;;DAAr$c.[R"U5$r/)rSentencePieceProcessor_bos_idrSs rbos_idSentencePieceProcessor.bos_id/rkr$c.[R"U5$r/)rSentencePieceProcessor_eos_idrSs reos_idSentencePieceProcessor.eos_id2rkr$c.[R"U5$r/)rSentencePieceProcessor_pad_idrSs rpad_idSentencePieceProcessor.pad_id5rkr$c.[R"U5$r/)r-SentencePieceProcessor_serialized_model_protorSs rserialized_model_proto-SentencePieceProcessor.serialized_model_proto8sKKDQQr$c.[R"X5$r/)r#SentencePieceProcessor_LoadFromFilerargs r LoadFromFile#SentencePieceProcessor.LoadFromFile;sAA$LLr$c 6[R"XX#XEXgU5 $r/)r#SentencePieceProcessor__EncodeAsIds rrenable_sampling nbest_sizealphaadd_bosadd_eosreverseemit_unk_pieces r _EncodeAsIds#SentencePieceProcessor._EncodeAsIds>s,AA$okp|CN\] ]r$c 6[R"XX#XEXgU5 $r/)r&SentencePieceProcessor__EncodeAsPiecesrs r_EncodeAsPieces&SentencePieceProcessor._EncodeAsPiecesAs-DDTQ`nsFQ_` `r$c 6[R"XX#XEXgU5 $r/)r/SentencePieceProcessor__EncodeAsSerializedProtors r_EncodeAsSerializedProto/SentencePieceProcessor._EncodeAsSerializedProtoDs-MMdZiw|HOZhi ir$c 6[R"XX#XEXgU5 $r/)r.SentencePieceProcessor__EncodeAsImmutableProtors r_EncodeAsImmutableProto.SentencePieceProcessor._EncodeAsImmutableProtoGs-LLTYhv{GNYgh hr$c 6[R"XX#XEXgX5 $r/)r(SentencePieceProcessor__EncodeAsIdsBatch rins num_threadsrrrrrrrs r_EncodeAsIdsBatch(SentencePieceProcessor._EncodeAsIdsBatchJs-FFtR]pzDKV]n nr$c 6[R"XX#XEXgX5 $r/)r+SentencePieceProcessor__EncodeAsPiecesBatchrs r_EncodeAsPiecesBatch+SentencePieceProcessor._EncodeAsPiecesBatchMs-II$U`s}GNY`q qr$c 6[R"XX#XEXgX5 $r/)r4SentencePieceProcessor__EncodeAsSerializedProtoBatchrs r_EncodeAsSerializedProtoBatch4SentencePieceProcessor._EncodeAsSerializedProtoBatchPs2RRSW^i}GPWbiz zr$c 6[R"XX#XEXgX5 $r/)r3SentencePieceProcessor__EncodeAsImmutableProtoBatchrs r_EncodeAsImmutableProtoBatch3SentencePieceProcessor._EncodeAsImmutableProtoBatchSs2QQRV]h|FOVahy yr$c.[R"X5$r/)r!SentencePieceProcessor__DecodeIdsridss r _DecodeIds!SentencePieceProcessor._DecodeIdsVs??JJr$c.[R"X5$r/)r(SentencePieceProcessor__DecodeIdsAsBytesrs r_DecodeIdsAsBytes(SentencePieceProcessor._DecodeIdsAsBytesYsFFtQQr$c.[R"X5$r/)r$SentencePieceProcessor__DecodePiecesrrs r _DecodePieces$SentencePieceProcessor._DecodePieces\sBB4PPr$c.[R"X5$r/)r2SentencePieceProcessor__DecodeIdsAsSerializedProtors r_DecodeIdsAsSerializedProto2SentencePieceProcessor._DecodeIdsAsSerializedProto_sPPQU[[r$c.[R"X5$r/)r5SentencePieceProcessor__DecodePiecesAsSerializedProtors r_DecodePiecesAsSerializedProto5SentencePieceProcessor._DecodePiecesAsSerializedProtobsSSTXaar$c.[R"X5$r/)r1SentencePieceProcessor__DecodeIdsAsImmutableProtors r_DecodeIdsAsImmutableProto1SentencePieceProcessor._DecodeIdsAsImmutableProtoesOOPTZZr$c.[R"X5$r/)r4SentencePieceProcessor__DecodePiecesAsImmutableProtors r_DecodePiecesAsImmutableProto4SentencePieceProcessor._DecodePiecesAsImmutableProtohsRRSW``r$c0[R"XU5$r/)r&SentencePieceProcessor__DecodeIdsBatchrrrs r_DecodeIdsBatch&SentencePieceProcessor._DecodeIdsBatchksDDTP[\\r$c0[R"XU5$r/)r-SentencePieceProcessor__DecodeIdsAsBytesBatchrs r_DecodeIdsAsBytesBatch-SentencePieceProcessor._DecodeIdsAsBytesBatchnsKKDWbccr$c0[R"XU5$r/)r7SentencePieceProcessor__DecodeIdsAsSerializedProtoBatchrs r _DecodeIdsAsSerializedProtoBatch7SentencePieceProcessor._DecodeIdsAsSerializedProtoBatchqsUUVZalmmr$c0[R"XU5$r/)r6SentencePieceProcessor__DecodeIdsAsImmutableProtoBatchrs r_DecodeIdsAsImmutableProtoBatch6SentencePieceProcessor._DecodeIdsAsImmutableProtoBatchtsTTUY`kllr$c0[R"XU5$r/)r)SentencePieceProcessor__DecodePiecesBatchrs r_DecodePiecesBatch)SentencePieceProcessor._DecodePiecesBatchwsGGS^__r$c0[R"XU5$r/)r:SentencePieceProcessor__DecodePiecesAsSerializedProtoBatchrs r#_DecodePiecesAsSerializedProtoBatch:SentencePieceProcessor._DecodePiecesAsSerializedProtoBatchzsXXY]doppr$c0[R"XU5$r/)r9SentencePieceProcessor__DecodePiecesAsImmutableProtoBatchrs r"_DecodePiecesAsImmutableProtoBatch9SentencePieceProcessor._DecodePiecesAsImmutableProtoBatch}sWWX\cnoor$c 4[R"XX#XEU5$r/)r(SentencePieceProcessor__NBestEncodeAsIdsrrrrrrrs r_NBestEncodeAsIds(SentencePieceProcessor._NBestEncodeAsIdss&FFtS]ho{IJ Jr$c 4[R"XX#XEU5$r/)r+SentencePieceProcessor__NBestEncodeAsPiecesrs r_NBestEncodeAsPieces+SentencePieceProcessor._NBestEncodeAsPiecess&II$V`kr~LM Mr$c 4[R"XX#XEU5$r/)r4SentencePieceProcessor__NBestEncodeAsSerializedProtors r_NBestEncodeAsSerializedProto4SentencePieceProcessor._NBestEncodeAsSerializedProtos'RRSW_it{GUV Vr$c 4[R"XX#XEU5$r/)r3SentencePieceProcessor__NBestEncodeAsImmutableProtors r_NBestEncodeAsImmutableProto3SentencePieceProcessor._NBestEncodeAsImmutableProtos'QQRV^hszFTU Ur$c 6[R"XX#XEXgX5 $r/)r1SentencePieceProcessor__SampleEncodeAndScoreAsIds rr num_samplesrwor include_bestrrrrs r_SampleEncodeAndScoreAsIds1SentencePieceProcessor._SampleEncodeAndScoreAsIdss.OOPT\gpsDKV]n nr$c 6[R"XX#XEXgX5 $r/)r4SentencePieceProcessor__SampleEncodeAndScoreAsPiecesrs r_SampleEncodeAndScoreAsPieces4SentencePieceProcessor._SampleEncodeAndScoreAsPiecess.RRSW_jsvGNY`q qr$c 6[R"XX#XEXgX5 $r/)r=SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProtors r&_SampleEncodeAndScoreAsSerializedProto=SentencePieceProcessor._SampleEncodeAndScoreAsSerializedProtos.[[\`hs|PWbiz zr$c 6[R"XX#XEXgX5 $r/)r to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. nbest_size = {0,1}: No sampling is performed. nbest_size > 1: samples from the nbest_size results. nbest_size < 0: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. alpha: Soothing parameter for unigram sampling, and dropout probability of merge operations for BPE-dropout. num_threads: number of threads in batch processing (Default = -1, auto-detected) ) model_file model_protoN) $_sentencepiece_processor_init_native _out_type_add_bos_add_eos_reverse_emit_unk_piece_enable_sampling _nbest_size_alpha _num_threadsLoad) rr%r&out_typerrrrrrrrs rInitSentencePieceProcessor.InitsVD+40nmmm+-#k% { Z A#r$c Uc URnUc URnUc URnUc URnUc URnUc UR nUc UR nU c URn U c URn US:XaUbUS:Xd US:XdU c [S5eU b[U 5[La [S5e[U5[LaxU[LaURXXxXXEU5 $U[LaURXXxXXEU5 $US:XdUS:XaUR!XXxXXEU5 $US:XaUR#XXxXXEU5 $U[LaUR%XUXXEU5$U[LaUR'XUXXEU5$US:XdUS:XaUR)XUXXEU5$US:XaUR+XUXXEU5$[S R-U55e) aEncode text input to segmented ids or tokens. Args: input: input string. accepsts list of string. out_type: output type. int or str. add_bos: Add to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. nbest_size = {0,1}: No sampling is performed. nbest_size > 1: samples from the nbest_size results. nbest_size < 0: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. alpha: Soothing parameter for unigram sampling, and merge probability for BPE-dropout (probablity 'p' in BPE-dropout paper). num_threads: the number of threads used in the batch processing (Default = -1). TrraWhen enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. when "nbest_size = -1" , this method samples from all candidates on the lattice instead of nbest segmentations.num_threads must be intserialized_protorimmutable_protozunknown out_type={})r(r)r*r+r,r-r.r/r0 RuntimeErrorrintlistrrrrrrrrrrt) rinputr2rrrrrrrrs rEncodeSentencePieceProcessor.EncodesW@  >> -- -- --  --  //  %%    '' D j&8J!O&0Ao .    [ 1 <455 e  s?''O(-.Z Z s?**5+07^] ] ) )X-@33E49GVdf f ( (225387Uce e S  !&>S S S##EJ$)GnV V ' '8w+>,,UZ-2W~_ _ & &++EJ,1Gn^ ^ .55h? @@r$c 4UR"SU[S.UD6$Nr<r2r%r=rrr<kwargss rEncodeAsPieces%SentencePieceProcessor.EncodeAsPieces5 [[ =us =f ==r$c 4UR"SU[S.UD6$r@r=r:rCs r EncodeAsIds"SentencePieceProcessor.EncodeAsIds9rGr$c ,UR"SUSS.UD6$)Nr7rAr%r=rCs rEncodeAsSerializedProto.SentencePieceProcessor.EncodeAsSerializedProto=s [[ Lu/A LV LLr$c ,UR"SUSS.UD6$)Nr8rAr%rMrCs rEncodeAsImmutableProto-SentencePieceProcessor.EncodeAsImmutableProtoAs [[ Ku/@ KF KKr$c 8UR"SXU[SS.UD6$NTr<rrr2rr%rBrr<rrrDs rSampleEncodeAsPieces+SentencePieceProcessor.SampleEncodeAsPiecesE- [[Gu5"%tG?EGGr$c 8UR"SXU[SS.UD6$rTrIrVs rSampleEncodeAsIds(SentencePieceProcessor.SampleEncodeAsIdsJrYr$c 0UR"SXUSSS.UD6$)Nr7TrUr%rMrVs rSampleEncodeAsSerializedProto4SentencePieceProcessor.SampleEncodeAsSerializedProtoOs- [[Vu5"4dVNTVVr$c 0UR"SXUSSS.UD6$)Nr8TrUr%rMrVs rSampleEncodeAsImmutableProto3SentencePieceProcessor.SampleEncodeAsImmutableProtoTs- [[Uu5"3TUMSUUr$cZ^^^^^^^Tc TRmTc TRmTc TRmTc TRmTc TRmTc TR mTS::aSmUUUUUUU4Sjn[ U5[LaUV s/sH o"U 5PM sn $U"U5$s sn f)aNBestEncode text input to segmented ids or tokens. Args: input: input string. accepsts list of string. out_type: output type. int or str. add_bos: Add to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) nbest_size: nbest size rrc>T[LaTRUTTTTT5$T[LaTRUTTTTT5$TS:XdTS:XaTR UTTTTT5$TS:XaTR UTTTTT5$[ S5e)Nr7rr8zunknown out_type)r:rrrrrr9)rrrrrr2rrs r_encode3SentencePieceProcessor.NBestEncode.._encode}s s?''j(/'>S S s?**4+2GWnV V ) )X-@33D*4;Wg~_ _ ( (2243:GWn^ ^-..r$)r(r)r*r+r,r.rr;) rr<r2rrrrrrens ` `````` r NBestEncode"SentencePieceProcessor.NBestEncodeYs(  >> -- -- --  --  %% q // e $)*Eq E** U^+s B(c 4UR"SX[S.UD6$Nr<rr2r%)rhrrr<rrDs rNBestEncodeAsPieces*SentencePieceProcessor.NBestEncodeAsPieces'   6E'*6.466r$c 4UR"SX[S.UD6$rk)rhr:rms rNBestEncodeAsIds'SentencePieceProcessor.NBestEncodeAsIdsrpr$c ,UR"SXSS.UD6$)Nr7rlr%rhrms rNBestEncodeAsSerializedProto3SentencePieceProcessor.NBestEncodeAsSerializedProtos+   EE'9E=CEEr$c ,UR"SXSS.UD6$)Nr8rlr%rurms rNBestEncodeAsImmutableProto2SentencePieceProcessor.NBestEncodeAsImmutableProtos+   DE'8D to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) num_samples: How many samples to return (Default = 1) alpha: inverse temperature for sampling wor: whether to sample without replacement (Default = false) include_best: whether to include the best tokenization, requires wor=True (Default = false) rg?Frznum_examples must be positivez8When include_best is True, We must specify "wor = True".c *> T[LaT RUTTT TTTTT5 $T[LaT RUTTT TTTTT5 $TS:XdTS:XaT R UTTT TTTTT5 $TS:XaT R UTTT TTTTT5 $[ S5e)Nr7rr8zunknown output type)r:rrrrr r9) rrrrrrrr2rrrs rre._encodes s?00{E3P\18'7N\ \ s?33D+ucS_4;Wg~_ _ ) )X-@<> -- -- --  --        :;; cUVV22& e $)*Eq E** U^+s9Cc 6UR"SXU[S.UD6$Nr<rrr2r%)r~rrr<rrrDs rSampleEncodeAndScoreAsPieces3SentencePieceProcessor.SampleEncodeAndScoreAsPieces*  & &?USX03?7=??r$c 6UR"SXU[S.UD6$r)r~r:rs rSampleEncodeAndScoreAsIds0SentencePieceProcessor.SampleEncodeAndScoreAsIdsrr$c .UR"SXUSS.UD6$)Nr7rr%r~rs r%SampleEncodeAndScoreAsSerializedProto67 7**5EE  # #E 11r$c^^UU4Sjn[U5[LaUVs/sH oC"U5PM sn$U"U5$s snf)NcV>T(aTRU5$TRU5$r/rrrr with_offsetss r _normalize4SentencePieceProcessor.Normalize.._normalize& ++D1 1t$$r$rr;rr<rrrEs` ` r Normalize SentencePieceProcessor.Normalize?% e ',-u! 1 u--  .<c t0nUR5Hup4[U5X#'M URU5$r/)itemsrr!)rrD new_kwargskeyr s rOverrideNormalizerSpec-SentencePieceProcessor.OverrideNormalizerSpecs5j *#e* '  ) )* 55r$c"UR5$r/rErSs r piece_size!SentencePieceProcessor.piece_size     r$c"UR5$r/rrSs r vocab_size!SentencePieceProcessor.vocab_sizerr$c"UR5$r/rzrSs r __getstate__#SentencePieceProcessor.__getstate__  ( ( **r$cFUR5 URU5 gr/rTr!rrzs r __setstate__#SentencePieceProcessor.__setstate__ mmo ""#9:r$c"UR5$r/rrSs rrSentencePieceProcessor.__len__rr$c$URU5$r/)rJrIs rr"SentencePieceProcessor.__getitem__s ^^E ""r$cU(aU(a [S5eU(aURU5$URU5$)zOverwride SentencePieceProcessor.Load to support both model_file and model_proto. Args: model_file: The sentencepiece model file path. model_proto: The sentencepiece model serialized proto. Either `model_file` or `model_proto` must be set. z-model_file and model_proto must be exclusive.)r9r!r)rr%r&s rr1SentencePieceProcessor.Loads:  JKK ++K88   z **r$) r)r*r/r,r-r.r0r(r+) NNNNNNNNNNN)NNNNNNr/)r7)r8)krrr:r;rrrr rTrdelete_SentencePieceProcessorrr!r'r,r1r5r;rArErJrOrTrYr]rarerirnrrrvrzrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rrrrr!r:r3r=rErJrNrQrWr[r^rarhrnrrrvryr~rrrrrrrrrrrrrrrrrrrrr1r>r%r$rrrs-/IOdeGHj%CC___VK_SHLIHIIHFBBBBRM]`ihnqzyKRQ\b[a]dnm`qpJMVUnqzyLWZkY!"-Bd"#[|>>MLG G V U"   #'#7t6 6 E D'+%)%)%),0)-#'!%*.JZ? ? N M &)d\~,/C),CCCCC 26!!+; !# +r$rc.[R"U5$r/)rSetRandomGeneratorSeed)seeds rrrs  0 0 66r$c.[R"U5$r/)rSetMinLogLevel)rKs rrrs  ( ( ++r$c\rSrSr\"SSSS9rSr\r\ S5r \ S5r \ S 5r \ S 5r \ S 5r\ SS j5r\ SSj5rSrg )SentencePieceTraineric6URR5$r/rCrDs rrFSentencePieceTrainer.rHr$c8URRU5$r/rCrJs rrFrrLr$rMrNc[S5e)NzNo constructor defined)r)rr@rDs rrTSentencePieceTrainer.__init__s566r$c.[R"U5$r/)r%SentencePieceTrainer__TrainFromString)rs r_TrainFromString%SentencePieceTrainer._TrainFromStringsCCCHHr$c.[R"U5$r/)r"SentencePieceTrainer__TrainFromMapr@s r _TrainFromMap"SentencePieceTrainer._TrainFromMaps@@FFr$c.[R"X5$r/)r#SentencePieceTrainer__TrainFromMap2r@iters r_TrainFromMap2#SentencePieceTrainer._TrainFromMap2AA$MMr$c.[R"U5$r/)r#SentencePieceTrainer__TrainFromMap3rs r_TrainFromMap3#SentencePieceTrainer._TrainFromMap3sAA$GGr$c.[R"X5$r/)r#SentencePieceTrainer__TrainFromMap4rs r_TrainFromMap4#SentencePieceTrainer._TrainFromMap4rr$Nc Ub'[U5[La[RU5$SnSnSn0nUR 5H#upgUS;aUnMUS;aUnMU"U5XV'M% U(aEU(a[R UU5nO[R U5nURU5 gU(a[RXS5$[RU5$)zDTrain Sentencepiece model. Accept both kwargs and legacy string arg.Nc>[U5[La|[RSS:Xa [ 5nO [ 5n[ R"USS9nURUVs/sHn[U5PM sn5 UR5$[U5$s snf)zEncode value to CSV..rr)lineterminator) rr;sysrStringIOBytesIOcsvwriterwriterowrgetvalue)r frrKs rre,SentencePieceTrainer._Train.._encodesy ;$    a A % A A::a3& //515a3q651 2 U 2s#B)sentence_iteratorsentence_reader) model_writer) rrrrrrrwriterr) rrDrer r rrr r&s r_TrainSentencePieceTrainer._Trains T#Y#-#44S99 lj *# : :#  $ $,#EN*/ '  ,;;J;LN+-;;JG+;' %44ZS S%33J? ?r$c z[US9 [R"SSU0UD6 SSS5 g!,(df  g=f)N)ostreamrr%) _LogStreamrr)r logstreamrDs rTrainSentencePieceTrainer.Trains, i (##66v6 ) ( (s, :r%r/r)rrr:r;rrrTrr staticmethodrrrrrrrr>r%r$rrrs-/IOdeG7HIIGGNNHHNN**X77r$rc\rSrSr\"SSSS9r\rSr\ Rr Sr Sr S rS rS rS rS rSrSSjrSSjrSrSrSrg)SentencePieceNormalizeric6URR5$r/rCrDs rrF SentencePieceNormalizer.rHr$c8URRU5$r/rCrJs rrFrrLr$rMrNcX[R"U[R"55 gr/)r SentencePieceNormalizer_swiginitnew_SentencePieceNormalizerrSs rrT SentencePieceNormalizer.__init__ s77n>h>h>jkr$c.[R"X5$r/)r/SentencePieceNormalizer_LoadFromSerializedProtors rr!/SentencePieceNormalizer.LoadFromSerializedProto$sMMd__r$c.[R"X5$r/)r'SentencePieceNormalizer_LoadFromRuleTSV)rr9s rLoadFromRuleTSV'SentencePieceNormalizer.LoadFromRuleTSV'sEEdUUr$c.[R"X5$r/)r(SentencePieceNormalizer_LoadFromRuleName)rrs rLoadFromRuleName(SentencePieceNormalizer.LoadFromRuleName*sFFtRRr$c.[R"U5$r/)r.SentencePieceNormalizer_serialized_model_protorSs rrz.SentencePieceNormalizer.serialized_model_proto-sLLTRRr$c.[R"X5$r/)r$SentencePieceNormalizer_LoadFromFiler~s rr$SentencePieceNormalizer.LoadFromFile0sBB4MMr$c.[R"X5$r/)r"SentencePieceNormalizer__Normalizers rr"SentencePieceNormalizer._Normalize3s@@LLr$c.[R"X5$r/)r-SentencePieceNormalizer__NormalizeWithOffsetsrs rr-SentencePieceNormalizer._NormalizeWithOffsets6sKKDWWr$c0[R"XU5$r/)r&SentencePieceNormalizer__SetProtoField)rrr s r_SetProtoField&SentencePieceNormalizer._SetProtoField9sDDTQVWWr$Nct[U5 U(aURU5nOVU(aURU5nO=U(aURU5nO$U(aUR U5nO [ S5eU(a7UR SU5 UR SU5 UR SU5 gg)aInitialzie sentencePieceNormalizer. Args: model_file: The sentencepiece model file path. model_proto: The sentencepiece model serialized proto. rule_tsv: The normalization rule file in TSV format. rule_name: Pre-defined normalization name. add_dummy_prefix: add dummy prefix. escape_whitespaces: escape whitespaces. remove_extra_whitespaces: remove extra whitespaces. zno model is specifiedadd_dummy_prefixescape_whitespacesremove_extra_whitespacesN)%_sentencepiece_normalizer_init_nativerr!r&r*r9r:) rr%r&rule_tsv rule_namer=r>r?statuss rr3SentencePieceNormalizer.Init<s(,D1 "":. --k: %%h/ &&y1233  .0@A 02DE 68PQ r$c^^UU4Sjn[U5[LaUVs/sH oC"U5PM sn$U"U5$s snf)NcV>T(aTRU5$TRU5$r/rrs rr5SentencePieceNormalizer.Normalize.._normalizecrr$rrs` ` rr!SentencePieceNormalizer.Normalizebrrc"UR5$r/rrSs rr$SentencePieceNormalizer.__getstate__mrr$cFUR5 URU5 gr/rrs rr$SentencePieceNormalizer.__setstate__qrr$r%)NNNNFFFr/)rrr:r;rrrr rTrdelete_SentencePieceNormalizerrr!r&r*rzrrrr:r3rrrr>r%r$rrrs-/IOdeGHl%DD`VSSNMXX# %&+$RL+;r$rc.[R"U5$r/)r SetDataDir)data_dirs rrOrOys  $ $X ..r$)r)rcH0nURR5H]up#[R"SU5(dM#[R"SSU5R 5R SS5nX1U'M_ UR5Hup#[XU5 M g)z1Added snake_cased method from CammelCased method.z^[A-Z]+z(?[U5[La$US:dXR5:a [S5eT"X5$)Nrzpiece id is out of range.)rr:rr)rKrgfuncs r_func_batchnize.._funcs5 Aw#~1q5A$7 2 33 :r$ct>[U5[LaUVs/sH nT"X5PM sn$T"X5$s snfr/r)rrrgrcs r _batched_func!_batchnize.._batched_funcs8 CyD&) *ceDnc ** 4 +s5)rrZ)r[rrfrcrbs @@r _batchnizerhs% $ '$   )=)r$rT)rJrOrTrYr]rare) __version__ sentencepiece package_datac*\rSrSrSSjrSrSrSrg)riNcrXlURb$[RR5Ulggr/)rrstderrfilenoorig_stream_fileno)rrs rrT_LogStream.__init__s+L || # 1 1 3d r$cURb_[R"UR5Ul[R "URR 5UR5 ggr/)rosduprporig_stream_dupdup2rorSs r __enter___LogStream.__enter__sI ||VVD$;$;r%r$rrrs4 > r$r)>rr_swig_python_version_info __package__rrrbuiltinsr ImportErrorrr&r,r6rr8objectr@>ImmutableSentencePieceText_ImmutableSentencePiece_swigregisterr'ImmutableSentencePieceText_swigregisterr,ImmutableNBestSentencePieceText_swigregisterr#SentencePieceProcessor_swigregisterrrr!SentencePieceTrainer_swigregisterr$SentencePieceNormalizer_swigregisterrOrUrrsimportlib.resources importlibiorrr_rhrTr'r@rZr3r=Tokenizer Detokenizemset_random_generator_seedset_min_log_level_versionripathrr resourcesfilesrr%r$rrs&:#/ "[ L$L 33nMMNADDP667QR4f4p;;<[\J +VJ +\223IJ7,K76K7^001EFW;fW;v334KL/  *"(>'F'F$(?(H(H% ,B,G,GH-D-I-IJ"8"?"?$:$A$A! A #Q'   &'$%'(2"! 277<<I//55oFG XYY%sG;; H H