bCi.SrSSKrSSKrSSKrSSKrSSKJr SSKJrJ r J r SSK J r SSK Jr SSKJrJr SS KJr SS KJr SS KJr SS KJrJrJrJrJrJr S SKJ r SSK!J"r" SSK#J$r$J%r%J&r&J'r'J(r( \"5(aSSK)J*r* OSr*\RV"\,5r-\\.\/\ \.\ \.44"/SS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PS S\"5(aS!OS44PS"S\"5(aSOS44PS#PS$\"5(aS%OS\"5(aS&OS44PS'PS(S\"5(aSOS44PS)\"5(aS*OSS44PS+PS,PS-\"5(aS.OS\"5(aS/OS44PS0S1\"5(aS2OS44PS3PS4S\"5(aS5OS44PS6PS7PS8S\"5(aSOS44PS9S:\"5(aS;OS44PSS\"5(aS5OS44PS?S@\"5(aSAOS44PSBS\"5(aSOS44PSCPSD\"5(aSEOS\"5(aSFOS44PSGPSH\"5(aSOS\"5(aSOS44PSIS\"5(aSOS44PSJS@\"5(aSAOS44PSKS\"5(aSOS44PSLS\"5(aSOS44PSMPSN\"5(aSOOS\"5(aSPOS44PSQSR\"5(aSSOS44PSTS\"5(aS!OS44PSUS\"5(aS!OS44PSVS\"5(aSOS44PSWSX\"5(aSYOS44PSZS[\"5(aS\OS44PS]\"5(aS^OS\"5(aS_OS44PS`PSaS\"5(aS5OS44PSbPScPSdS@\"5(aSAOS44PSeS:\"5(aS;OS44PSfSg\"5(aShOS44PSi\"5(aSjOS\"5(aSkOS44PSl\"5(aSOS\"5(aSOS44PSm\"5(aSOS\"5(aSOS44PSn\"5(aSOS\"5(aSOS44PSo\"5(aSOS\"5(aSOS44PSpPSq\"5(aSOS\"5(aSOS44PSrSs\"5(aStOS44PSuSv\"5(aSwOS44PSxSy\"5(aSzOS44PS{S:\"5(aS;OS44PS|S\"5(aSOS44PS}S\"5(aSOS44PS~S\"5(aSOS44PS\"5(aSOSS44PSPS\"5(aS:OS\"5(aS;OS44PSS\"5(aS5OS44PSS\"5(aSOS44PS\"5(aSOSS44PSPSS\"5(aS;OS44PSS\"5(aSOS44PSPSS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aS5OS44PSS\"5(aS5OS44PSS\"5(aS5OS44PSS\"5(aS5OS44PSS\"5(aS5OS44PS\"5(aSOSS44PSS:\"5(aS;OS44PSS:\"5(aS;OS44PSS:\"5(aS;OS44PSS\"5(aSOS44PSPSS\"5(aS5OS44PSS:\"5(aS;OS44PSPSPSPSPSPSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aS5OS44PSS\"5(aSOS44PSPSS@\"5(aSAOS44PSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PSS:\"5(aS;OS44PSS:\"5(aS;OS44PSSX\"5(aSYOS44PS\"5(aSOS\"5(aSOS44PSS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PSPS\"5(aSOS\"5(aSOS44PSS\"5(aS5OS44PSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PSS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PSPSS\"5(aSOS44PS\"5(aSOSS44PSS\"5(aSOS44PSS\"5(aSOS44PS\"5(aSOSS44PS\"5(aSOS\"5(aSOS44PS\"5(aSOS\"5(aSOS44PSS@\"5(aSAOS44PSS\"5(aSOS44PSS\"5(aSOS44PSPS\"5(aS:OS\"5(aS;OS44PS\ "5(aSO\"5(aSOS\"5(a\ "5(dSOS44PS\ "5(aSO\"5(aSOS\"5(a\ "5(dSOS44PS\ "5(aSO\"5(aSOS\"5(a\ "5(dSOS44PS\ "5(aSO\"5(aSOS\"5(a\ "5(dSOS44PSS\"5(aSOS44PS\"5(aSOSS44PSS\"5(aSOS44PSS\"5(aGSOS44PGSS\"5(aS5OS44PGSS\"5(aS5OS44PGSS\"5(aS5OS44PGSGS\"5(aGSOS44PGSS\"5(aSOS44PGSS@\"5(aSAOS44PGS \"5(aGS OS\"5(aGS OS44PGS S\"5(aSOS44PGS S\"5(aSOS44PGSGS\"5(aGSOS44PGSPGSS\"5(aS5OS44PGSS\"5(aSOS44PGS\"5(aGSOS\"5(aGSOS44PGS\"5(aGSOS\"5(aGSOS44PGS\"5(aSOS\"5(aSOS44PGSS\"5(aSOS44PGSS\"5(aSOS44PGSS\"5(aS;OS44PGSS\"5(aSOS44PGSS\"5(aSOS44PGSS\"5(aSOS44PGSGS \"5(aGS!OS44PGS"S:\"5(aS;OS44PGS#S\"5(aSOS44PGS$S\"5(aSOS44PGS%S\"5(aSOS44PGS&PGS'\"5(aS1OS\"5(aS2OS44PGS(\"5(aS1OS\"5(aS2OS44PGS)PGS*\"5(aSOS\"5(aSOS44PGS+SR\"5(aSSOS44PGS,S\"5(aSOS44PGS-S\"5(aSOS44PGS.PGS/S\"5(aSOS44PGS0S\ "5(aSO\"5(aS5OS44PGS1\"5(aGS2OSS44PGS3PGS4S\"5(aSOS44PGS5SX\"5(aSYOS44PGS6SX\"5(aSYOS44PGS7SX\"5(aSYOS44PGS8SX\"5(aSYOS44PGS9SX\"5(aSYOS44PGS:SX\"5(aSYOS44PGS;SX\"5(aSYOS44PGSSX\"5(aSYOS44PGS?SX\"5(aSYOS44PGS@SX\"5(aSYOS44PGSAPGSBGSC\"5(aGSDOS44PGSE\"5(aSOS\"5(aSOS44PGSF\"5(aGSGOS\"5(aGSHOS44PGSI\"5(aGSJOS\"5(aGSKOS44PGSLGSM\"5(aGSNOS44PGSOS@\"5(aSAOS44PGSPS@\"5(aSAOS44PGSQPGSRGSS\"5(aGSTOS44PGSUS\"5(aSOS44PGSV\"5(aGSWOS\"5(aGSXOS44PGSY\"5(aGSWOS\"5(aGSXOS44PGSZ\"5(aSOS\"5(aSOS44PGS[\"5(aGS\OSS44PGS]\"5(aSOS\"5(aSOS44PGS^S\"5(aS5OS44PGS_\"5(aGS`OSS44PGSaPGSb\"5(aGScOSS44PGSdPGSeGSf\"5(aGSgOS44PGShS\"5(aSOS44PGSiS:\"5(aS;OS44PGSj\"5(aSOS\"5(aSOS44PGSk\"5(aSOS\"5(aSOS44PGSl\"5(aSOS\"5(aSOS44PGSmPGSnPGSoPGSpS\"5(aSOS44PGSq\"5(aGSrOS\"5(aGSsOS44PGSt\"5(aSOS\"5(aSOS44PGSuS\"5(aSOS44PGSvS\"5(aSOS44PGSwS\"5(aSOS44PGSxS\"5(aSOS44PGSyPGSz\ "5(aSOS\"5(a\ "5(dS5OS44PGS{PGS|PGS}PGS~PGSGS\"5(aGSOS44PGSS\"5(aSOS44PGS\"5(aGSOS\"5(aGSOS44PGSPGS\"5(aGSOSS44PGS\"5(aSOS\"5(aSOS44PGS\"5(aSOS\"5(aSOS44PGS\"5(aGSOS\"5(aGSOS44PGSS\"5(aSOS44PGS\"5(aSOS\"5(aSOS44PGS\"5(aSOS\"5(aSOS44PGS\"5(aSOS\"5(aSOS44PGS\"5(aSOS\"5(aSOS44P5r0\""\$\05r1\$Rd"5VVs0sHupX_M snnr3GS\.GS\ \4\S44GSjr5GSGS\ \.\Rl\.4GS\ \ \.\Rl\.4GS\7GS\ \7GS\ \8\.\.4GS\ \ \7\.4GS\ \.GS\7GS\.GS\8\.\44GSjjr9"GSGS5r:GSGS/r;gs snnf(zAuto Tokenizer class.N) OrderedDict)AnyOptionalUnion)is_mistral_common_available)PretrainedConfig)get_class_from_dynamic_moduleresolve_trust_remote_code)load_gguf_checkpoint)PreTrainedTokenizer)TOKENIZER_CONFIG_FILE) cached_fileextract_commit_hashis_g2p_en_availableis_sentencepiece_availableis_tokenizers_availablelogging)EncoderDecoderConfig)_LazyAutoMapping)CONFIG_MAPPING_NAMES AutoConfigconfig_class_to_model_typemodel_type_to_module_name!replace_list_option_in_docstrings)PreTrainedTokenizerFastaimv2 CLIPTokenizerCLIPTokenizerFastalbertAlbertTokenizerAlbertTokenizerFastalign BertTokenizerBertTokenizerFastarceeLlamaTokenizerLlamaTokenizerFastaria aya_visionCohereTokenizerFastbark)bart) BartTokenizerBartTokenizerFastbarthezBarthezTokenizerBarthezTokenizerFast)bartpho)BartphoTokenizerNbertzbert-generationBertGenerationTokenizer)z bert-japanese)BertJapaneseTokenizerN)bertweet)BertweetTokenizerNbig_birdBigBirdTokenizerBigBirdTokenizerFastbigbird_pegasusPegasusTokenizerPegasusTokenizerFast)biogpt)BioGptTokenizerNbitnetr) blenderbot)BlenderbotTokenizerBlenderbotTokenizerFast)zblenderbot-small)BlenderbotSmallTokenizerNblipzblip-2 GPT2TokenizerGPT2TokenizerFastbloomBloomTokenizerFastblt bridgetowerRobertaTokenizerRobertaTokenizerFastbros)byt5) ByT5TokenizerN camembertCamembertTokenizerCamembertTokenizerFast)canine)CanineTokenizerN chameleon chinese_clipclapclipclipseg)clvp) ClvpTokenizerN code_llamaCodeLlamaTokenizerCodeLlamaTokenizerFastcodegenCodeGenTokenizerCodeGenTokenizerFastcoherecohere2colpalicolqwen2Qwen2TokenizerQwen2TokenizerFastconvbertConvBertTokenizerConvBertTokenizerFastcpm CpmTokenizerCpmTokenizerFast)cpmant)CpmAntTokenizerNcsm)ctrl) CTRLTokenizerN)zdata2vec-audioWav2Vec2CTCTokenizerNz data2vec-textdbrxdebertaDebertaTokenizerDebertaTokenizerFastz deberta-v2DebertaV2TokenizerDebertaV2TokenizerFast deepseek_v2 deepseek_v3 deepseek_vldeepseek_vl_hybrid)dia) DiaTokenizerN diffllama distilbertDistilBertTokenizerDistilBertTokenizerFastdprDPRQuestionEncoderTokenizerDPRQuestionEncoderTokenizerFastelectraElectraTokenizerElectraTokenizerFastemu3ernieernie4_5 ernie4_5_moeernie_mErnieMTokenizer)esm) EsmTokenizerNexaone4falcon falcon_mambaGPTNeoXTokenizerFastfastspeech2_conformerFastSpeech2ConformerTokenizer)flaubert)FlaubertTokenizerN flex_olmofnet FNetTokenizerFNetTokenizerFast)fsmt) FSMTTokenizerNfunnelFunnelTokenizerFunnelTokenizerFastgemmaGemmaTokenizerGemmaTokenizerFastgemma2gemma3 gemma3_textgemma3n gemma3n_textgitglmglm4glm4_moeglm4v glm4v_moezgpt-sw3GPTSw3Tokenizergpt2 gpt_bigcodegpt_neogpt_neox)gpt_neox_japanese)GPTNeoXJapaneseTokenizerNgpt_ossgptj)zgptsan-japanese)GPTSanJapaneseTokenizerN)graniterJN) granitemoer)granitemoehybridr)granitemoesharedrzgrounding-dinogroupvitheliumherbertHerbertTokenizerHerbertTokenizerFast)hubertrxibertideficsidefics2idefics3 instructblipinstructblipvideointernvljambajanusjetmoe)jukebox)JukeboxTokenizerNzkosmos-2XLMRobertaTokenizerXLMRobertaTokenizerFastz kosmos-2.5layoutlmLayoutLMTokenizerLayoutLMTokenizerFast layoutlmv2LayoutLMv2TokenizerLayoutLMv2TokenizerFast layoutlmv3LayoutLMv3TokenizerLayoutLMv3TokenizerFast layoutxlmLayoutXLMTokenizerLayoutXLMTokenizerFastled LEDTokenizerLEDTokenizerFastliltllamallama4 llama4_textllava llava_nextllava_next_videollava_onevision longformerLongformerTokenizerLongformerTokenizerFastlongt5 T5TokenizerT5TokenizerFast)luke) LukeTokenizerNlxmertLxmertTokenizerLxmertTokenizerFastm2m_100M2M100Tokenizermambamamba2marianMarianTokenizermbartMBartTokenizerMBartTokenizerFastmbart50MBart50TokenizerMBart50TokenizerFastmegaz megatron-bert metaclip_2)zmgp-str)MgpstrTokenizerNminimax ministralMistralCommonTokenizermistralmistral3mixtralmllamamlukeMLukeTokenizerzmm-grounding-dino mobilebertMobileBertTokenizerMobileBertTokenizerFast modernbert moonshinemoshimpnetMPNetTokenizerMPNetTokenizerFastmptmramt5 MT5TokenizerMT5TokenizerFastmusicgenmusicgen_melodymvp MvpTokenizerMvpTokenizerFast)myt5) MyT5TokenizerNnemotronnezhanllb NllbTokenizerNllbTokenizerFastznllb-moe nystromformerolmoolmo2olmo3olmoez omdet-turbo oneformerz openai-gptOpenAIGPTTokenizerOpenAIGPTTokenizerFastoptowlv2owlvit paligemma)parakeet)ParakeetCTCTokenizerNpegasus pegasus_x) perceiver)PerceiverTokenizerN persimmonphiphi3phimoe)phobert)PhobertTokenizerN pix2structpixtralplbartPLBartTokenizer) prophetnet)ProphetNetTokenizerNqdqbertqwen2 qwen2_5_omni qwen2_5_vl qwen2_audio qwen2_moeqwen2_vlqwen3 qwen3_moe qwen3_nextqwen3_omni_moeqwen3_vl qwen3_vl_moe)rag) RagTokenizerNrealmRealmTokenizerRealmTokenizerFastrecurrent_gemmareformerReformerTokenizerReformerTokenizerFastrembertRemBertTokenizerRemBertTokenizerFast retribertRetriBertTokenizerRetriBertTokenizerFastrobertazroberta-prelayernorm)roc_bert)RoCBertTokenizerNroformerRoFormerTokenizerRoFormerTokenizerFastrwkv seamless_m4tSeamlessM4TTokenizerSeamlessM4TTokenizerFastseamless_m4t_v2 shieldgemma2siglipSiglipTokenizersiglip2smollm3speech_to_textSpeech2TextTokenizer)speech_to_text_2)Speech2Text2TokenizerNspeecht5SpeechT5Tokenizer)splinter)SplinterTokenizerSplinterTokenizerFast squeezebertSqueezeBertTokenizerSqueezeBertTokenizerFaststablelm starcoder2switch_transformerst5t5gemma)tapas)TapasTokenizerN)tapex)TapexTokenizerN)z transfo-xl)TransfoXLTokenizerNtvpudop UdopTokenizerUdopTokenizerFastumt5 video_llavaviltvipllava visual_bert)vits) VitsTokenizerNvoxtral)wav2vec2rx)z wav2vec2-bertrx)zwav2vec2-conformerrx)wav2vec2_phoneme)Wav2Vec2PhonemeCTCTokenizerNwhisperWhisperTokenizerWhisperTokenizerFastxclipxglm XGLMTokenizerXGLMTokenizerFast)xlm) XLMTokenizerNzxlm-prophetnetXLMProphetNetTokenizerz xlm-robertazxlm-roberta-xlxlnetXLNetTokenizerXLNetTokenizerFastxlstmxmodyosozambazamba2 class_namereturncUS:Xa[$[R5HaupX;dM [U5nUS;aUS:Xa[R "SS5nO[R "SU3S5n[ X05s $ [RR5H#nUHn[ USS5U:XdMUs s $ M% [R "S5n[XP5(a [ XP5$g![a Mf=f) Nr)rrr r z.tokenization_mistral_common transformers.ztransformers.models__name__) rTOKENIZER_MAPPING_NAMESitemsr importlib import_modulegetattrAttributeErrorTOKENIZER_MAPPING_extra_contentvalueshasattr)r module_name tokenizersmodule tokenizer main_modules d/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/auto/tokenization_auto.pytokenizer_class_from_namer's..&&#:#@#@#B  #3K@KAAjTlFl"001OQ_`"001[M1BDYZ v22$C(66==? #Iy*d3zA  $@)).9K{''{// "  s7 C== D  D pretrained_model_name_or_path cache_dirforce_downloadresume_downloadproxiestokenrevisionlocal_files_only subfolderc  U RSS5n U b+[R"S[5 Ub [ S5eU nU R S5n [ U[UUUUUUUUSSSU S9n U c[RS5 0$[X5n [U S S 9n [R"U 5nSSS5 U WS'U$!,(df  N=f) a= Loads the tokenizer configuration from a pretrained model tokenizer configuration. Args: pretrained_model_name_or_path (`str` or `os.PathLike`): This can be either: - a string, the *model id* of a pretrained model configuration hosted inside a model repo on huggingface.co. - a path to a *directory* containing a configuration file saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. cache_dir (`str` or `os.PathLike`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force to (re-)download the configuration files and override the cached versions if they exist. resume_download: Deprecated and ignored. All downloads are now resumed by default when possible. Will be removed in v5 of Transformers. proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated when running `hf auth login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. local_files_only (`bool`, *optional*, defaults to `False`): If `True`, will only try to load the tokenizer configuration from local files. subfolder (`str`, *optional*, defaults to `""`): In case the tokenizer config is located inside a subfolder of the model repo on huggingface.co, you can specify the folder name here. Passing `token=True` is required when you want to use a private model. Returns: `dict`: The configuration of the tokenizer. Examples: ```python # Download configuration from huggingface.co and cache. tokenizer_config = get_tokenizer_config("google-bert/bert-base-uncased") # This model does not have a tokenizer config so the result will be an empty dict. tokenizer_config = get_tokenizer_config("FacebookAI/xlm-roberta-base") # Save a pretrained tokenizer locally and you can reload its config from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") tokenizer.save_pretrained("tokenizer-test") tokenizer_config = get_tokenizer_config("tokenizer-test") ```use_auth_tokenNrThe `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.V`token` and `use_auth_token` are both specified. Please set only the argument `token`. _commit_hashF) rrrrrrrr _raise_exceptions_for_gated_repo%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errorsrz\Could not locate the tokenizer configuration file, will try to use the model config instead.zutf-8)encoding)popwarningswarn FutureWarning ValueErrorgetrrloggerinforopenjsonload)rrrrrrrrrkwargsr commit_hashresolved_config_filereaderresults rget_tokenizer_configrEsRZZ 0$7N!  A    uv v**^,K&%%'))..305  # rs %&:HK "W 56" 6(F> M 6 5s C CcX\rSrSrSrSr\\"\5S55r \ SSj5r Sr g) AutoTokenizeria This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when created with the [`AutoTokenizer.from_pretrained`] class method. This class cannot be instantiated directly using `__init__()` (throws an error). c[S5e)Nz}AutoTokenizer is designed to be instantiated using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method.)OSError)selfs r__init__AutoTokenizer.__init__s _  c  URSS5nUb<[R"S[5 UR S5b [ S5eXCS'URSS5nSUS'URS S5nURS S5nURS S5nUR S 5n UbSn [ R US5n U c,[ S USSRS[ 55S35eU upU(a$U b [U 5n O[RS5 U c [U 5n U c[ SU S35eU R"U/UQ70UD6$[U40UD6nSU;aUSUS'UR S5nSnSU;a9[US[[45(aUSnOUSR SS5nUc[U[ 5(dPU (a0[#X40UD6n[%USS9Sn[&R("S(0UD6nO[&R"U4S U0UD6nUR*n[-US5(aSUR.;aUR.SnUSLn[1U5[2;=(d/ USL=(a$ [U5SL=(d [US-5SLnU(aCU(a USbUSnOUSnSU;aUR5S5SnOSn[7XUUU5nU(aNU(aG[9WU40UD6n URSS5nU R;5 U R"U/UQ7S U0UD6$UbgSn U(a&UR=S5(dUS3n[U5n U c Un[U5n U c[ SWS 35eU R"U/UQ70UD6$[U[>5(a{[1UR@5[1URB5LaD[RS!URBRDS"UR@RDS#35 URBn[G[1U5RH5nUb`[2[1U5unnU(a U(dUcUR"U/UQ70UD6$UbUR"U/UQ70UD6$[ S$5e[ S%URDS&SRS'[255S35e))ae Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary. The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by falling back to using pattern matching on `pretrained_model_name_or_path`: List options Params: pretrained_model_name_or_path (`str` or `os.PathLike`): Can be either: - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co. - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. - A path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not applicable to all derived classes) inputs (additional positional arguments, *optional*): Will be passed along to the Tokenizer `__init__()` method. config ([`PretrainedConfig`], *optional*) The configuration object used to determine the tokenizer class to instantiate. cache_dir (`str` or `os.PathLike`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download the model weights and configuration files and override the cached versions if they exist. resume_download: Deprecated and ignored. All downloads are now resumed by default when possible. Will be removed in v5 of Transformers. proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. subfolder (`str`, *optional*): In case the relevant files are located inside a subfolder of the model repo on huggingface.co (e.g. for facebook/rag-token-base), specify it here. use_fast (`bool`, *optional*, defaults to `True`): Use a [fast Rust-based tokenizer](https://huggingface.co/docs/tokenizers/index) if it is supported for a given model. If a fast tokenizer is not available for a given model, a normal Python-based tokenizer is returned instead. tokenizer_type (`str`, *optional*): Tokenizer type to be loaded. trust_remote_code (`bool`, *optional*, defaults to `False`): Whether or not to allow for custom models defined on the Hub in their own modeling files. This option should only be set to `True` for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. kwargs (additional keyword arguments, *optional*): Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`, `additional_special_tokens`. See parameters in the `__init__()` for more details. Examples: ```python >>> from transformers import AutoTokenizer >>> # Download vocabulary from huggingface.co and cache. >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased") >>> # Download vocabulary from huggingface.co (user-uploaded) and cache. >>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased") >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*) >>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/") >>> # Download vocabulary from huggingface.co and define model-specific arguments >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base", add_prefix_space=True) ```rNrrrconfigT _from_autouse_fasttokenizer_typetrust_remote_code gguf_filezPassed `tokenizer_type` z3 does not exist. `tokenizer_type` should be one of z, c3$# UHov M g7fN.0cs r 0AutoTokenizer.from_pretrained..+s D,Cq,Csrzt`use_fast` is set to `True` but the tokenizer class does not have a fast version. Falling back to the slow version.zTokenizer class z is not currently imported.rtokenizer_classauto_maprF)return_tensorsFastrrz-- code_revisionz- does not exist or is not currently imported.z The encoder model config class: z3 is different from the decoder model config class: z. It is not recommended to use the `AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder specific tokenizer classes.zzThis tokenizer cannot be instantiated. Please make sure you have `sentencepiece` installed in order to use this tokenizer.z!Unrecognized configuration class z8 to build an AutoTokenizer. Model type should be one of c38# UHoRv M g7fr)rrs rrrs4[IZAZZIZsr)%rrrrrrrjoinrrwarningfrom_pretrainedr isinstancetuplelistr rr r for_modelrrrtypersplitr r register_for_auto_classendswithrdecoderencoder __class__rr)clsrinputsrrrrrrrrtokenizer_class_tupletokenizer_class_nametokenizer_fast_class_nametokenizer_configconfig_tokenizer_classtokenizer_auto_map gguf_path config_dicthas_remote_codehas_local_code class_ref upstream_repo_tokenizer_class_candidate model_typetokenizer_class_pytokenizer_class_fasts rrAutoTokenizer.from_pretrainedsZ $4d;  % MME zz'". l-7OHd+#|::j$/$4d;"JJ':DAJJ{+   %"O$;$?$?PT$U !$, .~.>>qyy D,C DDEQH ?T ; ,8&?@Y&ZONN=&"; "!1!5!56G!H! ) )*:6 FF%5j%A"%5j%A%E%EoW[%\" " )f&677 +,I _X^ _I"6yQV"WX`"aK'11@K@F'775IZ^dF&,%;%; "vz**&///Q%+___%E",D8f):: "$ . )*@AMZ,-Cf-LMUYY  .q1=.q1 .q1 y ) 5a 8 $ 9!.Racp!  0;IGdohnoO ?D1A  3 3 5"22-06J[_e $ /"O 6 ? ? G G/E.Fd,K)";?22 ^^F/V 0E0EF  !7Hf7V 4  4#5G5O+;;[o^dohnoo$: /0@0@/AB++/994[IZ4[+[*\\] _  rNcUcUc [S5eUb [U[5(a [S5eUb [U[5(a [S5eUbDUbA[U[5(a,URU:wa[SURSUS35eU[ R ;a[ UupEUcUnUcUn[ RXU4US9 g) ar Register a new tokenizer in this mapping. Args: config_class ([`PretrainedConfig`]): The configuration corresponding to the model to register. slow_tokenizer_class ([`PretrainedTokenizer`], *optional*): The slow tokenizer to register. fast_tokenizer_class ([`PretrainedTokenizerFast`], *optional*): The fast tokenizer to register. NzKYou need to pass either a `slow_tokenizer_class` or a `fast_tokenizer_classz:You passed a fast tokenizer in the `slow_tokenizer_class`.z:You passed a slow tokenizer in the `fast_tokenizer_class`.zThe fast tokenizer class you are passing has a `slow_tokenizer_class` attribute that is not consistent with the slow tokenizer class you passed (fast tokenizer has z and you passed z!. Fix one of those so they match!)exist_ok)r issubclassrr slow_tokenizer_classrrregister) config_classr+fast_tokenizer_classr) existing_slow existing_fasts rr,AutoTokenizer.registers  ',@,Hjk k  + ;OQh0i0iYZ Z  + ;OQd0e0eYZ Z ! ,$0/1HII$99=QQ['<<==MNbMcd!!  ,;; ;+<\+J (M#+'4$#+'4$""?` @` D)r)rrrr)NFNNNNF)rQsa' #''G3\?5<3*B"   H %&c5# 1M+N&NOc  '>'@'@#d  c  %?%A%A!t)@)B)B%  c  ?;R;T;T$7Z^_`c  #=T=V=V%9\`ab!c " " ;?c @ 2Ac D &@&B&B"*A*C*C&  Cc P /KbKdKd1GjnopQc R .Sc T D7N7P7P3VZ[\Uc V KWc X AYc Z /:Q:S:S#6Y]^_[c \ O$@$@ d(?(A(A$t  yc F /BYB[B[+>aefgGc J "*A*C*C&  Ic X '>'@'@#d  Wc f '>'@'@#d  ec r *sc v (B(D(D$$,C,E,E(4  uc B 'CZC\C\)?bfghCc D D3J3L3L/RVWXEc F T4K4M4M0SWXYGc H %?V?X?X';^bcdIc J &@W@Y@Y(<_cdeKc L )F]F_F_+BeijkMc P "<">">D&=&?&?"T  Oc \ .]c ^ 4K4M4M0SWXY_c ` *ac b ;cc d -I`IbIb/Ehlmnec f /:Q:S:S#6Y]^_gc h 'CZC\C\)?bfghic l (B(D(D$$,C,E,E(4  kc z $>$@$@ d(?(A(A$t  yc H $>$@$@ d(?(A(A$t  Gc V $>$@$@ d(?(A(A$t  Uc d !$>$@$@ d(?(A(A$t  cc p (qc t $>$@$@ d(?(A(A$t  sc @ -LcLeLe/HkopqAc D -5L5N5N1TX  Cc P 'CZC\C\)?bfghQc R /:Q:S:S#6Y]^_Sc T ?;R;T;T$7Z^_`Uc V d4K4M4M0SWXYWc X $8O8Q8Q 4W[\]Yc Z *D*F*F&DRVWX[c \ (]c ` #:#<#<$'>'@'@#d  _c l D7N7P7P3VZ[\mc n $:Q:S:S 6Y]^_oc r $0C0E0E ,4QU V qc x 2yc z t4K4M4M0SWXY{c | /:Q:S:S#6Y]^_}c ~ *c @ %@W@Y@Y'<_cdeAc D $>$@$@ d(?(A(A$t  Cc R $>$@$@ d(?(A(A$t  Qc ` $>$@$@ d(?(A(A$t  _c n $>$@$@ d(?(A(A$t  mc | $>$@$@ d(?(A(A$t  {c J $>$@$@ d(?(A(A$t  Ic V 9P9R9R"5X\]^Wc X 4K4M4M0SWXYYc Z $5L5N5N1TXYZ[c \ d9P9R9R5X\]^]c ^ 46M6O6O2UYZ[_c ` t:Q:S:S6Y]^_ac b *D*F*F&DRVWXcc d /:Q:S:S#6Y]^_ec f AXAZAZ*=`defgc h _=T=V=V&9\`abic j d6M6O6O2UYZ[kc l Bmc n T8O8Q8Q4W[\]oc p /:Q:S:S#6Y]^_qc r ?sc t -uc v 0wc x 6yc z 6{c | OD[D]D]-@cghi}c ~ o>U>W>W':]abcc @ D7N7P7P3VZ[\Ac B 'CZC\C\)?bfghCc D 3Ec F %AXAZAZ'=`defGc H T3J3L3L/RVWXIc J &@W@Y@Y(<_cdeKc L &@W@Y@Y(<_cdeMc N /BYB[B[+>aefgOc P G^G`G`0CfjklQc R &@W@Y@Y(<_cdeSc V $>$@$@ d(?(A(A$t  Uc b 41H1J1J-PTUVcc f $>$@$@ d(?(A(A$t  ec r 0sc v )C)E)E%4-D-F-F)D  uc B ;R;T;T7Z^_`C c D )F]F_F_+BeijkE c F -LcLeLe/HkopqG c H -LcLeLe/HkopqI c J +I`IbIb-EhlmnK c L 7N7P7P!3VZ[\M c N 'F]F_F_)BeijkO c R $>$@$@ d(?(A(A$t  Q c ` $>$@$@ d(?(A(A$t  _ c n $>$@$@ d(?(A(A$t  m c z #=T=V=V%9\`ab{ c | (BYB[B[*>aefg} c ~ .H_HaHa0Dgklm c @ -G^G`G`/CfjklA c B -LcLeLe/HkopqC c F !;!=!= 4%<%>%>!D  E c R *S c T %@W@Y@Y'<_cdeU c V *D*F*F&DRVWXW c X 43J3L3L/RVWXY c Z D4K4M4M0SWXY[ c \ )C)E)E%4QUVW] c ` $>$@$@ d(?(A(A$t  _ c n &@&B&B"*A*C*C&  m c z $@W@Y@Y&<_cde{ c | ?CZC\C\,?bfgh} c @ %-D-F-F)D   c L /M c P #=#?#?T'>'@'@#d  O c ^ /00)*D*F*F&D(?(A(AJeJgJg$mq   ] c p /00)*D*F*F&D(?(A(AJeJgJg$mq   o c B /00)*D*F*F&D(?(A(AJeJgJg$mq   A c T /00)*D*F*F&D(?(A(AJeJgJg$mq   S c d $>U>W>W&:]abce c f 'A'C'C#tTUg c h G^G`G`0Cfjkli c j -LcLeLe/Hkopqk c l ;R;T;T7Z^_`m c n t:Q:S:S6Y]^_o c p 46M6O6O2UYZ[q c r #=T=V=V%9\`abs c t 1H1J1J-PTUVu c v #?V?X?X%;^bcdw c z "<">">D&=&?&?"T  y c F m:Q:S:S%6Y]^_G c H ]AXAZAZ,=`defI c J 7N7P7P!3VZ[\K c L *M c N d9P9R9R5X\]^O c P ?;R;T;T$7Z^_`Q c T #=#?#?T'>'@'@#d  S c b #=#?#?T'>'@'@#d  a c p %?%A%A!t)@)B)B%  o c | $2I2K2K.QUVW} c ~ 43J3L3L/RVWX c @ 40G0I0I,tTUAc B 43J3L3L/RVWXCc F  5L5N5N1TX Y Ec L ?V?X?X(;^bcdMc P  !?V?X?X#;^b c Oc V 9P9R9R"5X\]^Wc X ?;R;T;T$7Z^_`Yc Z O$@$@ d(?(A(A$t  Kc X #?V?X?X%;^bcdYc Z "U>W>W&:]abc]c ^ 0_c `  aefgKc L )CZC\C\+?bfghMc P  (?(A(A$t  Oc \ &@W@Y@Y(<_cde]c `  (?(A(A$t  _c n  (?(A(A$t  mc |  (?(A(A$t  {c H ,F]F_F_.BeijkIc J &@W@Y@Y(<_cdeKc L *D[D]D],@cghiMc N (Oc P #=T=V=V%9\`abQc T $>$@$@ d(?(A(A$t  Sc b 'A'C'C#+B+D+D'$  ac p &@&B&B"*A*C*C&  oc | +I`IbIb-Ehlmn}c ~ 'CZC\C\)?bfghc B # ;R;T;T!7Z^ _ Ac H 1Ic J )F]F_F_+BeijkKc L $2I2K2K.QUVWMc P *D*F*F&D.E.G.G*T  Oc ^ *D*F*F&D.E.G.G*T  ]c l $>$@$@ d(?(A(A$t  kc x )C)E)E%4QUVWyc | $>$@$@ d(?(A(A$t  {c H T8O8Q8Q4W[\]Ic J 6P6R6R2X\^bcdKc L >Mc N -G-I-I)tUYZ[Oc P EQc T  #CZC\C\%?bf g Sc Z d6M6O6O2UYZ[[c \ @W@Y@Y)<_cde]c ` "!;!=!= 4%<%>%>!D  _c n !;!=!= 4%<%>%>!D  mc | $>$@$@ d(?(A(A$t  {c H ,Ic J ,Kc L 5Mc N 9P9R9R"5X\]^Oc R #=#?#?T'>'@'@#d  Qc ` !;!=!= 4%<%>%>!D  _c l )CZC\C\+?bfghmc n /:Q:S:S#6Y]^_oc p &@W@Y@Y(<_cdeqc r AXAZAZ*=`defsc t *uc x ,G,I,I(t-D-F-FOjOlOl)rv  wc D 5Ec F :Gc H ?Ic J DKc L 'CZC\C\)?bfghMc N ?;R;T;T$7Z^_`Oc R #=#?#?T'>'@'@#d  Qc ^ (_c ` 8R8T8T4Z^`defac d )C)E)E%4-D-F-F)D  cc r )C)E)E%4-D-F-F)D  qc @ $>$@$@ d(?(A(A$t  c L 43J3L3L/RVWXMc P )C)E)E%4-D-F-F)D  Oc ^ %?%A%A!t)@)B)B%  ]c l $>$@$@ d(?(A(A$t  kc z $>$@$@ d(?(A(A$t  yc e N%%9;RS#7#=#=#?@#?41!$#?@#%S 42H@9= &*(,(,""l#(bkk#.>)>#?lc2;;s#3345lld^ l d38n % l E$)$ % lsmlll #s(^l^\r\r~  0_ AsZ.A]