oir SSKJr SSKJr SSKJr SSKrSSKrSSKJr SSK J r SSK r SSK r SSK r SSKrSSKrSSKrSSKrSSKJrJrJr SSKJr /S Qr\"S 5rS S /r\"\Vs/sHoR;5PM sn\Vs/sHoR;5S -PM sn-5rSR=\5\R>S'SSR=\R>RA55-r!S\!;r"S\!;r#Sr$C!S&Sjr%Sr&S'Sjr'Sr(Sr)Sr*S'Sjr+Sr,S(Sjr-S(Sjr.Sr/Sr0Sr1S)S jr2SSK3r3SS!K3J4r4 SSK5r5SSK6r5SS"K67 SS"K77 SS#K6J8r8 S%r9gs snfs snf! S$r8N=f)* AutoTokenizer)convert_slow_tokenizer)PreTrainedTokenizerFastN)logger)PeftModelForCausalLM)mean_of_trained_tokensadd_new_tokensfix_untrained_tokens)fix_zero_training_loss)load_correct_tokenizerfix_sentencepiece_tokenizercheck_tokenizerr fix_sentencepiece_gguf)CodeLlamaTokenizerFastCodeLlamaTokenizerz#unsloth/Qwen2.5-Coder-1.5B-Instructz!unsloth/Qwen2.5-Coder-7B-Instructz -bnb-4bit UNSLOTH_IGNORED_TOKENIZER_NAMESz COLAB_z KAGGLE_z/tmpc[US5(a URnO [U5nUR5nSnU(dXC;aUR USS5n[ U5nUVs/sH3ofR S5(dMURS5S:XdM1UPM5 nnUHn[XS5n U cM[XS-S5n SU S 3n URU 5[U 5-n U S :XaMNURS U 5n XA B; ##D%0$3/  #YKr2H$+eWB/I/77)QO 9+Rz3HE7"XJa0I/77)QO /"2*223CDO 9Ws*E0E0E0c[UR55nUR5VVs0sHup#X2_M nnn0nUH nXBnX%U'M U$s snnfN)sortedvaluesitems) dictionary sorted_keyskeyvalueinverted_dictionarysorted_dictionarys r;get_sorted_dictrHysg**,-K8B8H8H8JK8J*#5:8JK#(#&%   LsAc[USS5nU(aU$URRnUR5nUR S5(a"US[ S5*n[ SUSUS35nO[nURnXwRS5Sn[R"SU[RS 9nUV s/sHoR S 5(aMU PM nn [RnXwRS5Sn[R"SU[RS 9n U V s/sHoR S 5(aMU PM n n [[X-55n0n UHn [X S5X'M [US S 9U S 'U"S0U D6n [!UR#55n[!U R#55nX:HnUR$U R$:HnU(aU(dU$['X 5(d'[USS 9U S 'U"S0U D6n ['X 5(dU$UR(R+SS5n[,R.R1U5(d[,R2"U5 USU3nUR5U5 U R5U5 [6R8"U5n ['X 5(aU $U$! [nGNj=fs sn fs sn f)Nis_fastFr)z!__import__(f"transformers.models.z").FastzArgs:z\n[\s]+([^\s]{1,}) \(flags_fileT)r*tokenizer_object/r)r% __class____name__lowerr#r'evalr__doc__r&refindall MULTILINElistsetr<rH get_vocaball_special_tokensassert_same_tokenization name_or_pathr!ospathexistsmakedirssave_pretrainedrfrom_pretrained)slow_tokenizertemporary_locationrJtokenizer_namelowered_tokenizer_name class_name FastTokenizerdocsargsr/args2kwargsargfast_tokenizersorted_slow_tokenizersorted_fast_tokenizer check_vocab check_specialname new_locations r;convert_to_fast_tokenizerrxsni7G 0'11::!/!5!5!7 ! * *; 7 7/0C3{3C2CDJ 3JBRRVWM4M  D  '"$ %D ::.bll KD 7t!::g#6AtD 7 # * *D  '"$ %D JJ/r|| LE 91ZZ%8QE 9 DL! "D Fn48 !2>T!RF ",V,N,N,D,D,FG+N,D,D,FG'@K))^-N-NN m $N C C%6~QV%W!"&00'GG! !  & & . .sC 8D 77>>, - - &'()4&1L""<0""<0#22<@N?? q0/ 8 :s*AK6KK*!K*4K/K/K'a{% if messages[0]['role'] == 'system' %}{% if messages[1]['role'] == 'user' %}{{ '[INST] ' + messages[0]['content'] + ' ' + messages[1]['content'] + ' [/INST]' }}{% set loop_messages = messages[2:] %}{% else %}{{ '[INST] ' + messages[0]['content'] + ' [/INST]' }}{% set loop_messages = messages[1:] %}{% endif %}{% else %}{% set loop_messages = messages %}{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}a{% if messages[0]['role'] == 'system' %}{% if messages[1]['role'] == 'user' %}{{ '[INST] <> ' + messages[0]['content'] + ' <> ' + messages[1]['content'] + ' [/INST]' }}{% set loop_messages = messages[2:] %}{% else %}{{ '[INST] ' + messages[0]['content'] + ' [/INST]' }}{% set loop_messages = messages[1:] %}{% endif %}{% else %}{% set loop_messages = messages %}{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'].strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'].strip() + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}cl^[TS5(dg[T5n[[SU4SjU555n[[ UTR -55nSR S5nUVs/sH ofU:wdM UPM nnSnSnSn U=(a U=(a U n SRU5S-SRU5-n T"U 5RU"U 5R:Hn U =(a U $s snf! TR5RS S 5S [;aU s$g =f) Nr]Tc3># UH@nURS5(dMURS5S:XdM2[TU5v MB g7frrrN)r#r$r%).0r/rfs r; +assert_same_tokenization.. sE "A::h'+,-GGCLA,=+**"sA A A s�utf-8rz5A quick brown fox jumps over the lazy dog!! Hi r(rrF) rr"rZfilterr[r]decodejoin input_ids__repr__splitIGNORED_TOKENIZER_CHECKING) rfrqr.special_tokensr]replacement_charr/check_chat_template1check_chat_template2check_chat_template3check_chat_templatestringcheck_special_tokenss ` r;r^r^s` >#7 8 8N#I  "   Nc.>3T3T"TUV399'B%7Q%7@P;P!%7Q $L N!5N:N  II( )I Jgg() *  6 " , ,v0F0P0P P #;';;wRx  " " $ * *3 21 59S S& &s4 C;C;%AD/D3c zSSKJn U"5n[RRU5(d[R"U5 [RRUS35(dU$UR!U5 WR#5n U R%['US3S 5R)55 UR!U5 UR+5HoupU"U /S S 9R,n U Sn [/U 5S :wa[1S U SU SU S35 MBU Sn U R2U nUR4U :XdeXlMq ['US3S5nUR7U R955 SSS5 SSKJn UR>"UUR@URBS9nU$![a^nSSKnSSKJn U"UR R5n X"S5:a[SU S35eSnAGN! SSK J n SnAGN =fSnAff=f! GMI=f!,(df  N=f)Nr)import_protobuf)Versionz3.20.3z!Unsloth: Your protobuf version = zR is too new. Please downgrade via `pip install --force-reinstall protobuf==3.20.3`sentencepiece_model_pb2/tokenizer.modelrbF)add_special_tokensrz Skip mapping z to z since z is already in the tokenizer!wbr) eos_token pad_token)"#transformers.convert_slow_tokenizerr Exceptiongoogle.protobufunsloth_zoo.utilsrprotobuf __version__ RuntimeErrortransformers.utilsrr`rarbrcisfilerd ModelProtoParseFromStringopenreadrArr'printpiecespiecewriteSerializeToString transformersrrerr) old_tokenizer new_tokenizer token_mappingrgrregooglerprotobuf_versiontokenizer_file old_token new_tokenidstokenizer_piecefilerr)s r;rr[sFCG"1"3 77>>, - - &' 77>>/00@A B B!!"45,779N"" "##3 4d;@@B !!"45!. 3 3 5 YKeDNN!f s8q=  {$yk Khi  !f ,33C8O$$ 111 )!!6& #$$45t < >3356 =+--!++!++I A  C C " 1&v'B'BC '("33"78H7IJ\]4  C B B C^   = .SentencePieceTokenTypesirrQN) rS __module__ __qualname____firstlineno__NORMALUNKNOWNCONTROL USER_DEFINEDUNUSEDBYTE__static_attributes__rQr;SentencePieceTokenTypesrs  rrrrz/added_tokens.jsonrr)encodingc US$)NrrQ)items r;(fix_sentencepiece_gguf..sT!Wr)rDrzUnsloth: Extending zP/tokenizer.model with added_tokens.json. Originally tokenizer.model is of size (z6). But we need to extend to sentencepiece vocab size (z).g@r)'copyrrrjsonenumrrr`rarrrrr'rloaddictr?rAnparrayrZr@diffminmaxrwarningzipkeysencoderscorertypeextendrr)saved_locationrrrrrrsentence_piece_sizeradded_tokens_jsonnew_sizeadded_tokens_idsr new_tokensr added_tokens r;rrsu :'-779N 77>>^,,<= > >"" / 0$7<<>n334 77>>^,,>? @ @  23SW MQU IIdO N " &&(0DE#S):%;;Hxx%6%=%=%? @A 77# $D xxzQ$((*/!44 NN n-.22E1FG>>FZr K .//5E1F0F0HIJJ"%j2H2H2J"K %,,W5 ! 0== #L   ,  014 8D >3356 9  M N M@ 9 8  sJ% J  J J.c [(aUnO2[(a%[RR [ U5nOSnSn[ R"UUUUUSSSUS9 n[U5[LaSn[ R"UUUUUUS9nU(a U[;aU$SUR5;aU$SUR5;aU$Ub[US5(a"[US5(aURUl [US5(a"[US5(aURUl[Xx5(aU$[ R""S US 35 [%U5$U$! SnGN=f) NFT)model_max_length padding_sider2trust_remote_codeuse_fastlegacy from_slow cache_dir)rrr2rrmistralzphi-4 add_bos_token add_eos_tokenzUnsloth: Will load z as a legacy tokenizer.)IS_COLAB_ENVIRONMENTIS_KAGGLE_ENVIRONMENTr`rar KAGGLE_TMPrrerboolIGNORED_TOKENIZER_NAMESrTrrrr^rrrx) rhrrr2rr fix_tokenizerrfrqs r;_load_correct_tokenizerrs  GGLLY7  N&66 /' 1!  & Nt#"22+#- N N.EE n**, , N((* *  # >? 3 3 O9 9 ,:+G+GN ( >? 3 3 O9 9 ,:+G+GN ( $N C C! ! NN%n%55LM -^< <_s E..E5c [UUUUUUUS9n[USS5nS[[USS55R5;aUn O;UbSU;aSU;aSU;a S U;aUn O[ U5n UbU c [ S 5eXlU$) N)rhrrr2rrr chat_templaterr_rz[/INST]z[INST] bos_tokenrzHUnsloth: Fixing chat template failed - please file a report immediately!)rr%strrTfix_chat_templaterr) rhrrr2rrrr)old_chat_templaters r;r r Fs('+#-%I ?DAC >2>?EEGG)  % * * ) ) , , , ,) *)4  (]-BZ  + rcvURU5nURU5nX4s=:XaS:Xag X4:aU$U$)Nr)r&)templateendforendif where_endfor where_endifs r;_find_end_positionr vsA==(L--&K(b()  #  rcSnSn[XU5nUcSnSn[XU5nUcU$URU5nX[U5-SnURS5(aSOSnSU-S -U;aSU-S -U;axURS 5(abUR S 5(aLUR S 5S :Xa7UR S 5S :Xa"SU-S-U-U-nUSU[U5-U-nU$)Nz {% endfor %}z {% endif %}z {%- endfor %}z {%- endif %}z{%--rz{%z ifz set z{{z}}rz if add_generation_prompt %})r r&r' startswithr#r$)rrr  chosen_endwhere after_endfordashs r;_fix_chat_templaters( F E#M5AJ ' uE    z *E Z!8!:;L''..3BD t e</ 4K' ! 5  # #D ) )  ! !$ ' '   t $ )   t $ ) 4K8 8< G% O &&?J(?@<O rcH[USS5nUcgSnSSS./nURUSSS9 SnUcU$U(dSSS.OSSS./nURUSSS9nURUS SS9nXE:XaS U;alS U;af[U5nS U;a,S U;a&[S URS URS35e[ R "SURS35 UnU$[S URS35eU$! SSS./nURUSSS9 S nN! SnN=f=f)Nruserz Who are you?)rolecontentF)add_generation_prompttokenizehuman)fromrETz{% if add_generation_prompt %}z{%- if add_generation_prompt %}zUnsloth: The tokenizer `zz` does not have a {% if add_generation_prompt %} for generation purposes. Please file a bug report to the maintainers of `z ` - thanks!zUnsloth: We successfully patched the tokenizer to add a {% if add_generation_prompt %} to the chat_template. This is not a bug, but please notify the maintainers of `z` has a {% if add_generation_prompt %} for generation purposes, but wasn't provided correctly. Please file a bug report immediately - thanks!)r%apply_chat_templaterrr_r warning_once)r)r is_sharegptmessagesnoyesnew_chat_templates r;rrsI=MK 7  %% e &   N3 7H  & &%E ' B  ' '$5 ( C y -M A1F!3= A 08II5=NN".y/E/E.FGGGPG]G]F^^ik ##PPYPfPfOggrt!2   *9+A+A*BCAA  u   >:H  ) )%E * K Ks#C99D!<DDD!D!c pUR5RSS5S[;aU$URRR R SnURnUR5V V s0sHupU [U 5_M nn n [U5n U V s0sHoX_M nn [UR55GHupX:dM [UR55U Sn[UR55U SnU(GdURR5Vs/sHn[U5PM nnURnSSKn[#UR$R'SUR5555nUVs/sH nUU;dM UPM nnUVs/sH%nUUR(R5;dM#UPM' nn[+U5[+U5:H=(a [+U5[+U5:HnSn/nU(d[-U5nSU5nUVs/sHnU[/UUS54PM nn/nUH9nUH0unnUU:XdMUR1U5 UR1U5 M2 M; [+U5[+U5:HnU(aSnUnU(a[U5Hfun nUR(UnUR2U UR(U U(dM8WU U:XdMC[5UUU S5 [5UUU S-S5 Mh [7URR55U:a0[8R:"S US US US US 3 5 [=U5s $[?SUSUS US US3 5e[@(d [B(aSn OSn [DRF"UUUUSSSU S9n[IUUUUUUSS9s $ [=U5$s sn n fs sn fs snfs snfs snfs snf! [8R:"S5 Uss $=f)Nrrrc3P# UHn[U5[LaU/OUv M g7fr>)rrr|r/s r;r}"check_tokenizer..s%2>UtAw#~14>Us$&Fc3# UH5oRS5(dMURS5S:XdM1Uv M7 g7fr{)r#r$r&s r;r}r'*s0#(aJJx,@QWWS\UVEV5s ?? ?Trz#Unsloth loaded a broken tokenizer `z$`, but managed to repair it! Tokens z with ids z exceeds the max vocab size of zX. We removed these bad tokens. If you think this is incorrect, fix your tokenizer first.zUnsloth tried to load `z`, but cannot succeed. Tokens zG. Fix your tokenizer since it'll perform out of bounds memory accesses.huggingface_tokenizers_cache)rrr2rrrr)modelr) model_namerrr2_reloadzUnsloth: Tokenizer is most likely buggy, and Unsloth failed to repair it. It will still work, but beware of out of bounds memory accesses. Please file an issue on the model owner's repo about this issue.)%rrrr* embed_tokensweightshapeadded_tokens_decoderrArr? enumeraterrZr@special_tokens_map itertools frozensetchain from_iterable_added_tokens_encoderr'r"r%append_added_tokens_decodersetattrrrrrxrrrrrer)!r*r)r+rrr2r,max_embedding_sizeadded_tokens_fastindexrErCrDj bad_indices bad_tokensr/ added_tokensrr3can_be_removed1can_be_removed2can_be_removedremove_generic try_mappernamesgeneric_tokens try_removal name_token check_tokenr7 remove_idrs! r;rrs !!#q)!,0JJ1188>>qA!66.?.E.E.G.Gles5z.G*+K@KL /44 L/4467  &05578L>S>S>U2" /9"TjA^{1~/J#Iz!}dC#Iz!}u/DdK)C999>>@ADVV++A*N&&0\K=Hghzg{|uu 9CC#-j\:(\K=@_`r_st\] $#'<'<:   !)99'7#/!$" $ )  '! )!+'7#/!#A8f %Y //sM Y#U#&&WN !##W ! s<:O8$O>6P P(P4"P P :P?)PP5) getsource)*)neftune_post_forward_hookc0UR(a[R"URS5URS5-5nUR[R "U5- nU[R "U5RU*U5-nU$)a  Implements the NEFTune forward pass for the model using forward hooks. Note this works only for torch.nn.Embedding layers. This method is slightly adapted from the original source code that can be found here: https://github.com/neelsjain/NEFTune Simply add it to your model as follows: ```python model = ... model.embed_tokens.neftune_noise_alpha = 0.1 model.embed_tokens.register_forward_hook(neftune_post_forward_hook) ``` Args: module (`torch.nn.Module`): The embedding module where the hook is attached. Note that you need to set `module.neftune_noise_alpha` to the desired noise alpha value. input (`torch.Tensor`): The input tensor to the model. output (`torch.Tensor`): The output tensor of the model (i.e. the embeddings). rr)trainingtorchtensorsizeneftune_noise_alphasqrt zeros_likeuniform_)moduleinputoutputdimsmag_norms r;rOrOsp, ??<< AQ ?@D11EJJt4DDHe..v6?? 8TTF rc ^[S5n[[RR5nSGHunn[ X5(dM[ [SU355nURS5mURS5nSRU4SjU55nSnURS5nSRU4S jU55nUR5S-nUcf[R"S U3S -U[R[R-S 9n[U5S :XaMUS nUR!X3U-5nOUR!X5U-5nUVs/sH ofU;dM UPM snn[#SSRU5S3[%55 [#U[%5['55 [#SUSU3['55 GM SHnSup#[ [SUSU355nURS5mURS5nSRU4SjU55nSnUS- nUS- nUS- nURS5nSRU4SjU55nUR!X5U-5n[#U['55 [#SUSUSU3['55 M g! g=fs snf)z" Patches the trainer with changes z"trl.trainer.sft_trainer.SFTTrainerN))_prepare_non_packed_dataloaderN)_prepare_datasetNz sft_trainer.defrc3,># UH oTSv M g7fr>rQr|r/rs r;r}.patch_sft_trainer_tokenizer..91uvYa if 'tokenizer' not in locals(): tokenizer = processing_class if 'formatting_func' not in locals(): raise RuntimeError('Unsloth: Please file a bug report - `formatting_func` does not exist!') if 'dataset_text_field' not in locals() and 'args' in locals(): dataset_text_field = args.dataset_text_field if 'dataset_text_field' not in locals(): raise RuntimeError('Unsloth: Please file a bug report - `dataset_text_field` does not exist!') test_text = dataset[0][dataset_text_field] if (formatting_func is None and dataset_text_field is not None) else formatting_func(dataset[0])[0] chat_template = getattr(tokenizer, 'chat_template', None) chat_template = '' if chat_template is None else chat_template has_bos_token_already = (test_text.startswith(tokenizer.bos_token) or tokenizer.bos_token in chat_template) if getattr(tokenizer, 'bos_token', None) is not None else False if 'add_special_tokens' not in locals() and has_bos_token_already: from functools import partial tokenizer = partial(tokenizer, add_special_tokens = False) processing_class = tokenizer else: add_special_tokens = False if has_bos_token_already else add_special_tokens c34># UH nST-U-v M g7f NrQrcs r;r}rdC 1sU{Q zdef z\(.*?\).*?\:\nrLrz%from trl.trainer.sft_trainer import (r)z#trl.trainer.sft_trainer.SFTTrainer.z = )zsft_trainer.SFTTrainerzdpo_trainer.DPOTrainerzkto_trainer.KTOTrainer)trainz#if resume_from_checkpoint is False:z trl.trainer..c3,># UH oTSv M g7fr>rQrcs r;r}rdrerfa import subprocess, re, gc, numpy as np a = np.array([0,]) try: a = subprocess.check_output('nvidia-smi --query-gpu=memory.used --format=csv', shell = True) a = re.findall(rb'([\d]{1,})[\s]{1,}M', a) a = np.array([int(x.decode('utf-8'))/1024 for x in a]) except: if not torch.cuda.is_available(): raise RuntimeError('Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!') if ((a - PRE_CHECK) >= 1).sum() > 1: raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!') for _ in range(3): gc.collect() torch.cuda.empty_cache() pass tokenizer = self.processing_class if hasattr(self, 'processing_class') else self.tokenizer fix_untrained_tokens(self.model, tokenizer, self.train_dataset, IGNORED_TOKENIZER_NAMES, eps = 1e-16) fix_zero_training_loss(self.model, tokenizer, self.train_dataset) ao try: gradient_accumulation_steps = self.args.gradient_accumulation_steps if type(gradient_accumulation_steps) is int and gradient_accumulation_steps > 1: from transformers import __version__ as transformers_version from packaging.version import Version if Version(transformers_version) <= Version('4.45.2'): print('**** Unsloth: Please use our fixed gradient_accumulation_steps by updating transformers, TRL and Unsloth!\n'\ '`pip install --upgrade --no-cache-dir --no-deps unsloth transformers git+https://github.com/huggingface/trl.git`') except: pass a if hasattr(self, 'neftune_hook_handle'): self.neftune_hook_handle.remove() if hasattr(self, 'neftune_hook_handle'): del self.neftune_hook_handle if getattr(self, 'neftune_noise_alpha', None) is not None: self.model.get_input_embeddings().neftune_noise_alpha = self.neftune_noise_alpha self.neftune_hook_handle = self.model.get_input_embeddings().register_forward_hook(neftune_post_forward_hook) pass a if hasattr(self.train_dataset, 'column_names'): column_names = set(self.train_dataset.column_names) check = ['chosen', 'rejected', 'prompt', 'chosen_input_ids', 'chosen_attention_mask', 'chosen_labels', 'rejected_input_ids', 'rejected_attention_mask', 'rejected_labels', 'prompt_input_ids', 'prompt_attention_mask'] if all(x in column_names for x in check): self.train_dataset = self.train_dataset.remove_columns(['chosen', 'rejected', 'prompt']) del check, column_names c34># UH nST-U-v M g7frhrQrcs r;r}rdFrjrk)rUr"trltrainer sft_trainerrrMr&rrrstriprWrXrYDOTALLr'r!execlocalsglobals) rs all_imports function_namereplacerfunction check_textr/path_to_trainerrs @r;patch_sft_trainer_tokenizerrs?A ckk--.K  {22 TL"@AB e$>>$'99999 b & %%d+ YYC CC &&(4/  zz}o&):: ryy0H 8}!{H''Z2GHH''x2GHH# 5 1H}Q 5 4SXXa[M CVXN Xvx+ 1-M? S I y D #Q TL0A=/"RST e$>>$'99999 T 0         %%d+ YYC CC ##H8.CD Xwy! ?+1]O3}o N I iS~ 6s K" K /K K)T)_unsloth_sentencepiece_temp)NrightNFr)T)zunsloth/llama-2-7b-bnb-4bitirNT):rrrrrrWr`(transformers.models.llama.modeling_llamarpeftrrRr3 collectionsnumpyrgc subprocesspsutilunsloth_zoo.tokenizer_utilsr r r unsloth_zoo.training_utilsr __all__r4rrrTrenvironrkeynamesrrrr<rHrxmistral_templatellama_templater^rrrr r rrrinspectrMrqtrl.trainer.sft_trainertransformers.trainerrOr)r/s0r;rs'F0 ;%     '*' $/0/1WWY/0(?@(?1wwy;(?@A15 :Q0R ,- $))BJJOO-. .!X-#x/   +\7J^20Vz7 L^E T .Ql .-`BJ`/ N0b %"A@f U1@Ts)E2E7 (E<<F