oiE* SSKJr SSKJr SSKJrJr SSKJ r J r J r J r J r Jr SSKJr SSKJr SSKJr SSKJrJrJrJr SSKrSSKrSSKrSSKrSSKrSSK r SSK!r!SS K"J#r# S S K$J%r%J&r&J'r' SSK(r(SSK)r)SSK*r*SS K"J#r# S S K+J,r, S S K-J.r. S SK/J0r0 S SK1J2r2J3r3 SSK4J5r5 SSK6J7r7 SSK6J8r8 SSK;Jr>J?r? /SQr@/SQrASSR\RR55-rES\E;rFS\E;rGSrHCESrISrJ0SS_SS _S!S"_S#S$_S%S&_S'S(_S)S*_S+S,_S-S._S/S0_S1S2_S3S4_S5S6_S7S8_S9S:_S;S<_S=S>_S?S@SASBSCSDSE.ErKSFrL\L"5(aSGOSHrMSIrNSSJjrOSKrPSLrQSMrR\RSNSOSSPS\RSQSPSSPSSRSSOSSSSSTSU4SV\\U\R4SW\USX\WSY\\\U\W4SZ\WS[\\XS\\S]\\Y\U4S^\WS_\\US`\WSa\\WSb\\USc\\WSd\WSe\USf\USg\\USh\USi\Z4(Sjjj5r[Skr\Slr]/4Smjr^SSnjr_SSojr`SSpjraSqrbSSr\USs\USt\USu\WSv\USw\USx\WSy\W4SzjjrcSS{SOSSPS\RSQSPSSPSSTS|4SV\\U\R4SW\USX\WSY\\\U\W4SZ\WS[\\XS\\S]\\Y\U4S^\WS_\\US`\WSg\\USh\USi\Z4S}jjrdSS~\USW\USa\\WSb\\USc\\WSY\\W\US4S]\\Y\US4Sd\WS^\WSe\USf\USg\\\USh\USi\Z4SjjreSrfSrgSSjrhSSjriSrjSrkS\USr\US\US\U4SjrlS\USr\US\U4SjrmS\USr\US\U4SjrnSSSSOSSSPS\RSQSPSSPSSTS4SV\\U\R4Sw\USX\WSY\\\U\W4Sc\\WSZ\WS[\\XS\\S]\\Y\U4S^\WS_\\US`\WSg\\USh\USi\Z4SjjroSS~\USw\USa\\WSb\\USc\\WSY\\W\US4S]\\Y\US4Sd\WS^\WSe\USf\USg\\\USh\USi\Z4SjjrpSrqSS~\USa\\WSb\\USc\\WSY\\W\US4Sd\WSe\USf\USh\USi\Z4SjjrrSSV\USh\USi\Z4SjjrsS S K-J.r. SSKtJuruJvrv SSKJ r J rw \RSSj5rx\RSSNSOSSPS\RSQSPSSPSSRSSOSSSSSTSU4SV\\U\R4SW\USX\WSY\\\U\W4SZ\WS[\\XS\\S]\\Y\U4S^\WS_\\US`\WSa\\WSb\\USc\\WSd\WSe\USf\USg\\USh\USi\Z4(Sjj5rySS{SOSSPS\RSQSPSSPSSTS|4SV\\U\R4SW\USX\WSY\\\U\W4SZ\WS[\\XS\\S]\\Y\U4S^\WS_\\US`\WSg\\USh\USi\Z4SjjrzSS~\USW\USa\\WSb\\USc\\WSY\\W\US4S]\\Y\US4Sd\WS^\WSe\USf\USg\\\USh\USi\Z4Sjjr{SSV\\U\R4SX\WSY\\\U\W44Sjjr|SSV\\U\R4SX\WSY\\\U\W44Sjjr}SSV\\U\R4SX\WSY\\\U\W44Sjjr~SrSSjrg! SSK9J8r8 GNK! SSK:J8r8 GNW=f=f))Version)version)dtype_from_configHAS_TORCH_DTYPE)convert_to_gguf quantize_ggufuse_local_ggufinstall_llama_cppcheck_llama_cpp_download_convert_hf_to_gguf) Linear4bit)Linear)OptionalCallableUnionListN)logger)fast_dequantize QUANT_STATEget_lora_parameters_bias)fix_sentencepiece_gguf)get_model_name)_convert_torchao_model)OLLAMA_TEMPLATESMODEL_TO_OLLAMA_TEMPLATE_MAPPER)ProcessorMixinHfApi) get_token)Path)PeftModelForCausalLM PeftModel)print_quantization_methodsunsloth_save_model save_to_ggufpatch_saving_functionscreate_huggingface_repo)zllama-quantizez llama-cliz llama-server z COLAB_z KAGGLE_z/tmp)zself_attn.q_projzself_attn.k_projzself_attn.v_projzself_attn.o_projz mlp.gate_projz mlp.up_projz mlp.down_proj)input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormzself_attn.q_normzself_attn.k_norm not_quantizedz8Recommended. Fast conversion. Slow inference, big files.fast_quantizedz9Recommended. Fast conversion. OK inference, OK file size. quantizedz:Recommended. Slow conversion. Fast inference, small files.f32zINot recommended. Retains 100% accuracy, but super slow and memory hungry.bf16zNBfloat16 - Fastest conversion + retains 100% accuracy. Slow and memory hungry.f16zNFloat16 - Fastest conversion + retains 100% accuracy. Slow and memory hungry.q8_0z=Fast conversion. High resource use, but generally acceptable.q4_k_mzZRecommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_Kq5_k_mzZRecommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_Kq2_kzWUses Q4_K for the attention.vw and feed_forward.w2 tensors, Q2_K for the other tensors.q3_k_lzTUses Q5_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_Kq3_k_mzTUses Q4_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_Kq3_k_szUses Q3_K for all tensorsq4_0zOriginal quant method, 4-bit.q4_1z`Higher accuracy than q4_0 but not as high as q5_0. However has quicker inference than q5 models.q4_k_szUses Q4_K for all tensorsq4_kzalias for q4_k_mzalias for q5_k_mz )ALLOWED_QUANTSitemsprint)keyvalues rKr$r$s,$**,  #geW%&-rJc[US5(dgURnSn[RR XR 5nSn[RR U5(dSn[R"U5 URU5 [RRUS35(aSnU(a[R"USS9 U$)N_saved_temp_tokenizerFTz/tokenizer.model ignore_errors) hasattrrUospathjoin name_or_pathexistsmakedirssave_pretrainedisfilerGrmtree)modeltemporary_locationtemp_tokenizersentencepiece_model file_locationcreated_folders rKcheck_if_sentencepiece_modelrhs 51 2 200NGGLL!35P5PQMN 77>>- ( ( M"""=1 ww~~'7899" mT: rJcxSSKJn [U"5R5nUHnURUR R :XdM)[UR5SRnU"5RU5n[R"SUR-5 UR5 M g)Nr)scan_cache_dirz-Unsloth: Will remove a cached repo with size )huggingface_hubrjlistreposrepo_idconfig _name_or_path revisions commit_hashdelete_revisionsr warning_onceexpected_freed_size_strexecute)rbrj cached_repos cached_reporemove_cache_commitdelete_strategys rK_free_cached_modelr{s.(../L$   %,,"<"< <"&{'<'<"=a"@"L"L ,.??#O   ?!99:   # # %$rJcz[USS5n[U[[[45(Ga[ U5up4pVprUb/[ U5[La UROUSn[X45nO URnUR[R5R5nUbURUR5R[R5UR5R[R5US9 [R"UR!5R#5UR55n [R$"U 5R'5(d[)SUS35eUR5RU5nX24$UR*nX24$)Nbias)alphazUnsloth: Merge failed. z has some elements = infinity.)getattr isinstanceBnb_Linear4bitPeft_Linear4bit Peft_Linearrtyperldtypertotorchfloat32taddmm_maxminabsisfiniteitem ValueErrorweight) layernamer}W quant_stateABsrmaximum_elements rK _merge_lorarsS 5&$ 'D%./;GHH(@(G%a  "%)+%6d%B !! TU  /AGGE DD  ! ! # = HHQSSUXXemm,accehhu}}.EqH Q$ii quuw?O>>/27799 .tf4RS CCEHHUO 7N LL 7NrJcP[SUS35 [R"UU5 g)NzUnsloth: Saving ...)rQrsave)shardrs rKfast_save_picklers+ TF# &' JJ   rJloraFT5GBTrained with Unsloth+Upload model trained with Unsloth 2x faster _unsloth_temporary_saved_buffersg?save_directory save_method push_to_hubtokenis_main_process state_dict save_functionmax_shard_sizesafe_serializationvariantsave_peft_format use_temp_dircommit_messageprivate create_prrevisioncommit_descriptiontagsrcmaximum_memory_usagecUc [5nUcSnSU;aUS- nUR5nUcSnO SU;aUS- nUS:Xa [S5eUS :XaSn[[ 55nS HnUU M U(aS S KJn U"US 9SnUS :aUS::de[S5H6n[RR5 [R"5 M8 UR5RSS5nUS:waUS:waUS:wa [S5eUS:XaB[S5 [S5 [!US5(aUR#5n[S5 Ub-[%U[&[(45(de['U5S/-nOS/nUUS'US:XdUS:XaU(aUc [S5eUS:Xa [S5 OUS:Xa [S5 [+UUUS S!SSUS"9n[-US#UR.5"UU UUUU UU UUUS$9 Ub?UR0nS%Ul[-US#UR.5"UU UUUU UU UUUS$9 UUl[!US&5(a[S'US(3U-5 US4$US)SSUS*US+US,S-.nS S.KJn [%UU5(a UR6nOUnUS:Xd3US:Xd-[!US/5(a[!UR6S05(GdUS*S1LaS2OS3nUHnUU M [!US45(aUR9S/5 U(a[+UUS)US S!SSUS"9nUbB[S5SS69 UR0nS%UlUR:"Su0UD6 UUl[S75 O [5 [S8SS69 US:wa [S9SS69 US:XaSUS:'UR:"Su0UD6 U(a"[!US&5(a[S;US)-5 [S75 US4$SnU(aS(aV[@RBRE[FU U R=S<5S=-S5n [HRJ"S>US?U 35 O[HRJ"S@USAUS?U 35 U US)'U US)'U n[SB5 [LRN"5RPn!SCn"[SU 5[TLa[VRX"SDU [VRZSE9n#[VRX"SFU [VRZSE9n$U#(a$[]U#R_S=55SG-SG-SG-n"O(a$[@RBRE[FU5n[@RBRkU5(d[@Rl"U5 [>(d [n(a![HRJ"SS5 [qU5 S STK9J:n& U&"5n[wURx5n'[SU'5[TLa-U'SU:Xa[Rzn'OU'SV:Xa[R|n'UR6R~RRRU'5USW'[][RRS 5RU-5n([SX5 S SYKEJEn) [U)"UR6R55GH"un*n+[Hn,[SZU,35n-S[U*S\U,S]3n.[U-U.5un/n0U0b U0US[U*S\U,S^3'[RR5U/R-U(:aU/UU.'Mn[HRJ"S_5 [@RBREUU.S`35n1[R"U/U1[[RSa9 [R"U1SbSIS1Sc9UU.'M [Hn,[SZU,Sd35US[U*S\U,S]3'M GM% UR6RRRUSe'UR6R~RR5URRR5:wa2URRRRU'5USf'UR5Hlun2n3[!U3Sg5(aU3R=UU2'n3[SU35[RLdMF[HRJ"ShU2Si[SU35S\35 Mn UUSj'U(dS2OS3nUHnUU M [!US45(aUR9S/5 U(a[+UUS)US S!SUUS"9nUS)nUS*(a%[UUU5un n4UbS S KJn U"US 9Sn5OUn5US*(aUW5:wa[SkW 35 S1US*'U US)'UbB[S5SS69 UR0nS%UlUR:"Su0UD6 UUl[S75 O [5 URxn6URxR5n7SlU7;aU7Sl Un8[SURx5RU75n7[!U8S/5(a&U8R6n8U7U8l<[!U8S/5(aM&U7Ul<US*(arUW5:wal[SkW 35 U US)'S1US*'UR:"Su0UD6 [@R"U 5n9[US,S 9n:[Sm5 U:RU S\U S/SnSoSp9 OUR:"Su0UD6 Un8[!U8S/5(a&U8R6n8U6U8l<[!U8S/5(aM&U6Ul<[S5 U(aC[!US&5(a2[SqUS This is the fastest and easiet. Just saves LoRA modules. "merged_16bit" ==> This merges LoRA weights and saves to float16. Needed for llama.cpp / GGUF. "merged_4bit" ==> This merges LoRA weights and saves to 4bit. Useful for DPO / inference.z1Unsloth: Merging 4bit and LoRA weights to 4bit...zThis might take 5 minutes...merge_and_unloadzDone.unslothrzoUnsloth: Pushing to HF requires a token. Pass `token = 'hf_....'` Go to https://huggingface.co/settings/tokens.z-Unsloth: Saving LoRA adapters. Please wait...z7Unsloth: Saving 4bit Bitsandbytes model. Please wait... finetunedtrl)rf old_usernameroriginal_push_to_hub) rnrrrrrrrrrrleftrozSaved z! model to https://huggingface.co/rrrr)r legacy_formatfilename_prefixrrr)r"rblayersF)rrrrrr)rrrrradd_model_tagszUnsloth: Saving tokenizer...endz Done.zUnsloth: Saving model...z+ This might take 10 minutes for Llama-7b...selected_adaptersz Saved to https://huggingface.co//rzUUnsloth: You are pushing to hub in Kaggle environment. To save memory, we shall move  to zCUnsloth: You are pushing to hub, but you passed your HF username = z. We shall truncate z2Unsloth: Merging 4bit and LoRA weights to 16bit...lz([0-9]{1,})[\s]{0,}GB)flagsz([0-9]{1,})[\s]{0,}MBi)logicalTrr~zUnsloth: You have z CPUs. Using `safe_serialization` is 10x slower. We shall switch to Pytorch saving, which might take 3 minutes and not 30 minutes. To force `safe_serialization`, set it to `None` instead.rg?zUnsloth: Will use up to z out of z RAM for saving.zUnsloth: Kaggle/Colab has limited disk space. We need to delete the downloaded model which will save 4-16GB of disk space, allowing you to save on Kaggle/Colab.) OrderedDictfloat16bfloat16zmodel.embed_tokens.weightz6Unsloth: Saving model... This might take 5 minutes ...)tqdmzlayer.z model.layers..z.weightz.biasz& We will save to Disk and not RAM now.z.pt) pickle_modulepickle_protocolcpu) map_locationmmap weights_onlyz .weight.datazmodel.norm.weightzlm_head.weightdata Unsloth: z is not a Tensor but a rz-Unsloth: Saving to organization with address quantization_configz.Unsloth: Uploading all files... Please wait...(Trained with Unsloth)z*.md) folder_path path_in_reporn repo_typerignore_patternsz-Saved merged model to https://huggingface.co/ rVrI)`r lstrip RuntimeErrordictlocalsrkrrangercuda empty_cachegccollectlowerreplacerQrXrrrltupleupload_to_huggingfacerr padding_sidepeftr"rbrr_findIS_KAGGLE_ENVIRONMENTrYrZr[ KAGGLE_TMPrrtpsutilvirtual_memory availablerstrrematch IGNORECASEintgroup cpu_countrrroundtotalr]r^IS_COLAB_ENVIRONMENTr{ collectionsrrrorr embed_tokensrrrget_device_properties total_memoryr enumerater LLAMA_WEIGHTSevalrmemory_allocatednbytesrpickleHIGHEST_PROTOCOLloadLLAMA_LAYERNORMSnormdata_ptrlm_headrPTensor_determine_usernameto_dict from_dictlistdirr upload_foldersplitrGra) ) = ( i  m# AB ,- 5, - -**,E g $u ....Dz     (,V$  K=$@{ =@  & A B M ) K L "        -u/@/@A$'++!!3!3   (55 %+I " I5y7L7L M(+!/!!/%%7#%7 &6I " 5( # #  %FG.X t##33CD/ >+I6)'2 *%-..  % 6 !w''w~7K7KX/V/V( 6%?  $'H(2' 5* + +    %()9: $#! A  0 ; )55 %+I "  % % @(? @&6I " (O G (3 & ?r J & <@ $%8 9 9 89 75(33 2*+;<=  ht##Hsn,+%&D(:(?(?(DE/0B0G0G0Lq0P0RS !#./A/F/Fs/Ka/O/QR"    11?0@EWDXZ    UV^U_`%%3$4D9K8LN  6H !124F 01+ >?##%//G. Ns"88 $nbmm 88 $nbmm   #HNN1$5 6 = Dt K   #HNN1$5 6 = D  n  $*   .F ~!!# ~!!9K !56 1 )G H #( 9K !564A 1$$$t++#a/$889G " d"4' + ,H &&(..t3D8=q A BBR TWW\\*6HI 77>>, - - &' 4 4 `  >*(J#N$9$9:K KC ) #--K J &..K ))005588E*+ ((+88;OOH BC(k.*>*>*E*EFG5!D&(D"1#QtfG4D!$-GAt>B ]1#QtfE:; ++-8HD#$ 4 ##$MN77<<(:tfCLI $*&,&=&= $)::U4PU$ 4 ;"@%D @DTF,/A ]1#QtfG<=%CHT'5&:&:&?&?&F&F&K&KJ"# ))0099;  ! ! ( ( 1 1 3 4(6'='='D'D'I'I'L'L ( #$ !&&( U 5& ! !&+jj 0JsOe ;ell *   )C50GU }TU V W ).8\*  $# $X .#u&''    !  $%5 6    # ..>?N .+> He, (L   .$U3F;O&O .H4O =>P=QRS16 .4F 01 ,B7%11!' !!<$;<"2  h J%%'J * , -Nell#--j9J .' * *'-- * .' * *EL .H4O =>P=QRS6H !1227 /&&B)ABJJ12 7@A >?,(5$   &&B)ABN .' * *'-- * .' * *EL 'Nwuh// ;H:Q~G\G\]`GaGgGghkGlmoGpFq r .2\*$Z%5%5%78zcr/SQn[R"U[R[RS9nU$)N)gitclonez --recursivez&https://github.com/ggerganov/llama.cppstdoutstderr subprocessPopenDEVNULLSTDOUT) full_command run_installers rK$install_llama_cpp_clone_non_blockingrK\s6L $$z11J>4 5 5 77>>2 3 3 77>>. / / 77>>> ? ? 77>>8 9 9 Y   : @ 0 4 6 7rJczSS/n[RRS5(ag[U5 SS[R "5=(d SS-S3/n[U5S :XaQS [ 3S [R "5=(d SS-S S R[53SS/n[U5 gg)Nrwzpip install gguf protobufrPrMrxrr~ryrgrQrzr{rr|r}) rYrZr]rsrr rVr[rW)use_cudarns rKinstall_llama_cpp_blockingrs G#H ww~~k"" " v'').Q12-@ H8'ZZcYd e?AQAQASAXWXZ[@[?\\tuxu}u}PvQuR S 6 $   H(rJc[RRS5R[R5nUHnUHxn[R R X#5n[R RU5(dMH[R"U[R5(dMtUs s $ M g)NPATH) rYenvirongetr#pathseprZr[r]accessX_OK) executablessystem_directories directory executablerZs rKget_executabler!s{/55bjjA' %J77<< 6Dww~~d## $(@(@ &( rJ model_name model_type model_dtypeis_sentencepiecemodel_directoryfirst_conversionis_vlm is_gpt_ossc  [RRSS5S:XaSn OSn US:XdUS:XdeUS:XaSOS n[U[5(aOE[U[ 5(aU/nO,[U[ 5(a [ U5nO [S 5eUS :Xa;[RR5(d[R"S 5 SnUcUnUH$n U RS 5(dM[S5e /n UHn U S:XaUn OU S:XaSn OU S:XaSn OU cSn U [R!5;a:SU S3n [R#5HupU SU SUS3- n M [U 5eU R%U 5 M U nU(a['S5 SnS/nOUc[)U5S:Xa USS:XaSnOdSnUH?n U S:Xa[+US5nMU S:Xa[+US5nM+U S :XdM3[+US5nMA US:aSnOUS:aSnO US:aS nOS nUS :Xa;[RR5(d[R"S 5 SnUS:XaS!OUnS"[-S#5[-S#5S$[-S#5S%[-S#5S&US'[-S#5S(US)US*3n['U5 [/5unn['S+5 ['S.5 [55 [75unnn['S/US035 ['S15 [9UUUUUUUUUS2U S39 unnS S S 5 WnWHPn[R:R=U5(aM)[0(a[S4US535e[S4US635e ['S7U35 UR?5n[@RB"5nUcSnUS-nU(dUSnSnUHSn X:wdM ['S8US9U S:35 US;U RE5S<3n[GUUU UU S=9nUR%U5 SnMU ['SA5 U(a:URKU5 [MU5RO5 URQ5 O ['SB5 U(aSn!OU[SU5;n!['SC5 ['SDU35 UU!U4$! ['S,5 [0(a[3SU S-9unnGN[3SU S-9unnGN=f!,(df  GN=f![Ha.n [0(a[S>US?35e[S>US@35eS n A ff=f)Ezh Orchestrates the complete GGUF conversion process. Handles installation, conversion, and quantization. UNSLOTH_ENABLE_LOGGING01TFrrr3r2FUnsloth: quantization_method can only be a string or a list of stringszlUnsloth: Cannot convert to bf16 GGUF since your computer doesn't support it. We shall switch instead to f16.Niq2zGUnsloth: Currently iq2 type quantizations aren't supported yet - sorry!r.r/r4r0r5Unsloth: Quant method = [$] not supported. Choose from below: [] => r)zCUnsloth: GPT-OSS model detected - using special conversion settingsNonerrr1rr~z:Unsloth: Switching bf16 to f16 due to hardware limitationsrzA==((====))== Unsloth: Conversion from HF to GGUF information \z= /| [0] Installing llama.cpp might take 3 minutes. O^O/ z_/ z [1] Converting HF to GGUF z might take 3 minutes. z! / [2] Converting GGUF rz` might take 10 minutes each. "-____-" In total, you will have to wait at least 16 minutes. z>Unsloth: llama.cpp found in the system. Skipping installation.z;Unsloth: Installing llama.cpp. This might take 3 minutes...) gpu_support print_outputz&Unsloth: Preparing converter script...z#Unsloth: [1] Converting model into z GGUF format.zThis might take 3 minutes...50GB) r input_folderrquantization_typeconverter_locationsupported_text_archssupported_vision_archsrrrrzUnsloth: Conversion failed for z You are in a Kaggle environment with limited disk space (20GB). Try saving to /tmp for more space or use a smaller model. Alternatively, save the 16bit model first, then convert manually.z' Please check disk space and try again.z.Unsloth: Initial conversion completed! Files: zUnsloth: [2] Converting GGUF z into z. This might take 10 minutes...rz.gguf) input_gguf output_gguf quant_typequantizer_locationrz!Unsloth: Quantization failed for a You are in a Kaggle environment, which might be the reason this is failing. Kaggle only provides 20GB of disk space in the working directory. Merging to 16bit for 7b models use 16GB of space. This means using `model.{save_pretrained/push_to_hub}_merged` works, but `model.{save_pretrained/push_to_hub}_gguf will use too much disk space. You can try saving it to the `/tmp` directory for larger disk space. I suggest you to save the 16bit model first, then use manual llama.cpp conversion. Error: {e}a| You might have to compile llama.cpp yourself, then run this again. You do not need to close this Python program. Run the following commands in a new terminal: You must run this in the same folder as you're saving your model. git clone --recursive https://github.com/ggerganov/llama.cpp cd llama.cpp && make clean && make all -j Once that's done, redo the quantization. Error: {e}zUnsloth: Model files cleanup...z:Unsloth: GPT-OSS model - skipping additional quantizationsz5Unsloth: All GGUF conversions completed successfully!zGenerated files: )*rYrrrrlrr TypeErrorrris_bf16_supportedrwarning startswithrrOkeysrPappendrQlenrchrr rr r r rrZr]copyrr upperr Exceptionremover!unlinkreverse frozenset)"rrrrrquantization_methodrrrr quant_methodnew_quantization_methodserrorrRrSstrengthfirst_conversion_dtype print_inforrconverter_pathrr initial_files is_vlm_updatefileall_saved_locationsr0 base_ggufquants_createdoutput_locationquantized_fileewant_full_precisions" rKr&r&.sD  zz~~.4;   ) #{j'@@ @&)3%K%t,, ' - -   ' / /"#67 T  fUZZ%A%A%C%C .  &,  " "5 ) )Y , "+ ? *&L - -!L [ (#L  !!L ~224 4/ ~=bcE,224 1SEugR005u% % '' 5#,$3 ST!%h  #&'1,1DQ1G61Q#) $7L#u,#&x#3%.#&x#3%/#&x#3 %8q=',$]',$]'-$'-$6!%***F*F*H*HST #3v#=RCS "gYs2wi ByCG9$BCYBZZr r7)45K4LDQdPefO P *1@1B.. NO 23   ( * E,.D 12H1I W  ,.'6#*% 0!/#7%;##' ( $ } 2Fww~~d##$$"5dV%)N#0T /0   & &y 1 O " " $ ' ' ) JK".)](https://github.com/unslothai/unsloth) cSnURS5nSU;a3SSKJn U"US9Sn[U5[LaX1:waUnUSU3nX4$UR S5SnX4$! [ SUS 35e=f) Nrz./rrrrrrz is not a Huggingface directory.)rrkrrrrr#)rrrr&rs rKrrsH#**40N . * e,V4HL!S(X-E' (z>*:;N  ##"'',Q/  ##  N++KL s )A$$A5c:Uc [5n[USU5upSSKJn U"UUSSUS9 SSKJn [ R UURRURRSSS 9nU"U5nURXS 9 [US 9n X4$! N=f) Nrr create_reporbFrnrrexist_okr ModelCardrr& base_modelrmethodextrar) r rrkrr MODEL_CARDformatrorprrr) rbrrrr&rrcontentcardr=s rKr(r(s  } 2>2uMN+ $   .##3300 $ ! 75 !F  !!  s A&BBc F[XU5upSSKJn U "UUSSUS9 SSKJn [R U UR RUR RSUS9n U "U 5n U RXS 9 UGb[US 9nS U;aXURS 5S -SnOUnSSK nURS S S9n[U5S:a;[SUS -5 UH&nURUUR!US5USSS9 M( URUUUSSS9 U(ahSSKn[%SSSS9nUR'SUR R0USS9 SSS5 URSSUSSS9 [(R*"S5 U $! GN=f!,(df  NA=f)NrrrbFrrrrrrrz*out.tfevents*T) recursivez6Unsloth: Uploading tensorboard files... Please wait...r)path_or_fileobjrrnrrz_temporary_unsloth_config.jsonwrdencodingr)indent config.json)rrkrrrrrorprrrrfindglobrrQ upload_filerjsonopendumprYr)rbrrrrrfrr create_configr&rrrrr=uploaded_locationr ftevent_files ftevent_filerrs rKrrs 3>QVWN+ $   .##3300 $ ! 7 u% -  -.A.A#.F.J.L M  -   "2 E }  ! H 00 !. ""&2#/#7#7 r#J, '%= #!. +,$5    6PTX <)@)@A4RS TQ   "B,(#!9   II6 7 Oi  RQPsA&F /(F F F cSn[USS5nU"S5RS[USS5:XaUbURU;d,SURSS5;dS URSS5;aNS n[R "S 5 [ R"S SU5n[ R"S SU5nX0lX4$)NF chat_templaterr bos_token_idz {bos_token}rrz {bos_token+TzUnsloth: ##### The current model auto adds a BOS token. Unsloth: ##### Your chat template has a BOS token. We shall remove it temporarily.z2\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\}[\s]{0,}\}z0\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\+[\s]{0,}) r input_ids bos_tokenrrrrsubr)r fix_bos_tokenrnew_chat_templates rKfix_tokenizer_bos_tokenr=sMI=M~"gi&NN  $   = 0 5 5c2 >> 5 5c2 >> M NNe  !#Er=! !#C!!  '8 #  ''rJcd[R"U5nU(d[SUS35 g[R"U5nU(d[SUS35 gUUlUnSnSnSnSn UR SU5R S U5R S U5R S U 5nUR US5R US 5nS U;aUR UURS 9nOUR US9nUR SS 5R SS 5R5nU$)zc Creates an Ollama Modelfile. Use ollama.create(model = "new_ollama_model", modelfile = modelfile) z5Unsloth: No Ollama template mapping found for model 'z'. Skipping Ollama ModelfileNu)⚫@✅#🦥__FILE_LOCATION__⚡@🦥#⛵u%⚫@✅#🦥__EOS_TOKEN__⚡@🦥#⛵u ⚫@✅#🦥u ⚡@🦥#⛵z{__FILE_LOCATION__}z{__EOS_TOKEN__}{} __EOS_TOKEN__)__FILE_LOCATION__r)r) rrrQr_ollama_modelfilerr eos_tokenrstrip) rbase_model_namemodel_locationollama_template_nameollama_modelfile modelfileFILE_LOCATION_REPLACEREOS_TOKEN_REPLACERLEFT_BRACKET_REPLACERRIGHT_BRACKET_REPLACERs rKcreate_ollama_modelfiler ^sl ;>>O  COCTTp q '++,@A  COCTTp q !IH@*+ /1GH "$6 7 + , , - !! 5 g "34)#$$ .%//% $$ .% !!.#6>>~sSZZ\I rJr&tagmodelfile_pathc p[R"SS/SSSS9nURS:Xa$[URR 55 O [S5 [R"S S US US U3S U/[R[RSSSS9n[URRS5H+n[USS9 [RR5 M- UR5nUS:wa[SU35 g[S5 g![R a gf=f)NrFhttp://localhost:11434Trcapture_outputtexttimeoutrOllama Server is not RunningOllama Request Timeoutollamacreater:z-frrBrCr'rcuniversal_newlinesrrz' MODEL CREATED FAILED WITH RETURN CODE z MODEL CREATED SUCCESSFULLYrErunrkrQrBstripTimeoutExpiredrFrirHiterreadlinesysrhwait)r&rr!r" init_checkprocessrr return_codes rKcreate_ollama_modelr;s+ (^^ - .!   A % *##))+ , 0 1  j*Qse ,   ""! GW^^,,b1 d" 2,,.Ka 8 FG ,-7  $ $('(sA D DD54D5c j[R"SS/SSSS9nURS:Xa$[URR 55 O [S5 [R"S S US US U3/[R[RSS SS9n[URRS5H+n[USS9 [RR5 M- UR5nUS:wa[SU35 g[S5 g![R a gf=f)NrFr$Trr%rr)r*r+pushrr-rr.rrz) MODEL PUBLISHED FAILED WITH RETURN CODE z MODEL PUBLISHED SUCCESSFULLYr0)r&rr!r8r9rrr:s rKpush_to_ollama_hubr>s (^^ - .!   A % *##))+ , 0 1 6hZq AcU;<""! GW^^,,b1 d" 2,,.Ka :;-HI ./+  $ $('(sA D DD21D2c[XS9n[SU3SSS9nURU5 UR5 SSS5 [ UUUSU3S9 [ X#US9 [ S5 g!,(df  N3=f) N)r gguf_location Modelfile_rrdr)r&rr!r")r&rr!zSuccessfully pushed to ollama)r rwritecloser;r>rQ)rr@r&rr! model_filefs rKpush_to_ollamarFs}(J  :,' AQ     B%j\2 (3O )* B As "A'' A5333333?c  Uc [S5e[URRSS9nUR S5SnU(a [S5eSn[ US5(a`[ URS 5(aE[ S URR55nU=(d [ URS 5nU=(a [U[5n[ URS 5(aURRS :Xd5[ URS 5(aURRS;aSOSn[[55nUUS'UUS'SUS'U(aSUS'OSUS'US US US US US US US US U(a[UR5unnO[U5unn[SU(aSOSS 35 [!SG0UD6 U(a URnU(aUUl[)S"5H[nS#SKnUR,"5 [.R0R35(dM=[.R0R55 M] [7UR5nURRn[9U5[:LaUS$:XdUS%:XdeO9U[.R<:XaS$nO"U[.R>:XaS%nO [AS&5e[S)5 /nUb[U[B5(aOE[U[:5(aU/nO,[U[D5(a [CU5nO [AS*5e[GU5HUun n!U!RI5n!U!S+:XaS,n!OU!S-:XaS.n!OU!S/:XaS0n!OU!cS.n!URKU!RI55 MW [MUWUSUUUUUS19 un"n#n$Sn%Sn&U"(aU$(a[QUUS45n'O[QUUU"S#5n'U'bU$(a![RRTRWUS55n%O3[RRTRW[RRX"5S55n%[[U%S6S7S89n(U(R]U'5 SSS5 Sn&U(a[^R`"S:5 U$(a9[S;5 [S5 [S?5 O[S@U"S#SA35 U&(a U$(a[SBU%35 [SC5 U&(aU$(d[SD5 [SE5 UU"U%U#U$USF.$! URRnUR S5SnGNH=f!["an[%S!U35eSnAff=f!["an[S'US(35 S$nSnAGNSnAff=f!["a,n[N(a[%S2U35e[%S3U35eSnAff=f!,(df  GN=f!["an[S9U35 SnAGNSnAff=f)HaE Same as .save_pretrained(...) except 4bit weights are auto converted to float16 then converted to GGUF / llama.cpp format. Choose for `quantization_method` to be: "not_quantized" : "Recommended. Fast conversion. Slow inference, big files.", "fast_quantized" : "Recommended. Fast conversion. OK inference, OK file size.", "quantized" : "Recommended. Slow conversion. Fast inference, small files.", "f32" : "Not recommended. Retains 100% accuracy, but super slow and memory hungry.", "f16" : "Fastest conversion + retains 100% accuracy. Slow and memory hungry.", "q8_0" : "Fast conversion. High resource use, but generally acceptable.", "q4_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K", "q5_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K", "q2_k" : "Uses Q4_K for the attention.vw and feed_forward.w2 tensors, Q2_K for the other tensors.", "q3_k_l" : "Uses Q5_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_K", "q3_k_m" : "Uses Q4_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_K", "q3_k_s" : "Uses Q3_K for all tensors", "q4_0" : "Original quant method, 4-bit.", "q4_1" : "Higher accuracy than q4_0 but not as high as q5_0. However has quicker inference than q5 models.", "q4_k_s" : "Uses Q4_K for all tensors", "q4_k" : "alias for q4_k_m", "q5_k" : "alias for q5_k_m", "q5_0" : "Higher accuracy, higher resource usage and slower inference.", "q5_1" : "Even higher accuracy, resource usage and slower inference.", "q5_k_s" : "Uses Q5_K for all tensors", "q6_k" : "Uses Q8_K for all tensors", "iq2_xxs" : "2.06 bpw quantization", "iq2_xs" : "2.31 bpw quantization", "iq3_xxs" : "3.06 bpw quantization", "q3_k_xs" : "3-bit extra small quantization", N.Unsloth: Saving to GGUF must have a tokenizer.F) load_in_4bitrrz`Unsloth: Please use .push_to_hub_gguf() instead of .save_pretrained_gguf() with push_to_hub=Truero architecturesc3D# UHnURS5v M g7f)ForConditionalGenerationForVisionText2TextNendswith.0rs rK /unsloth_save_pretrained_gguf..?s$ . JJI J J.  vision_configGptOssForCausalLMr)zgpt-ossgpt_ossTrbrrmxfp4rrrrrrrrr is_processorz"Unsloth: Merging model weights to z16-bitz format...zFailed to save/merge model: rrrrz4Unsloth: Model dtype can only be float16 or bfloat16z$Unsloth: Could not determine dtype (z), defaulting to float16z%Unsloth: Converting to GGUF format...rr.r3r/r4r0r5) rrrrrrrrrzUnsloth: GGUF conversion failed in Kaggle environment. This is likely due to the 20GB disk space limit. Try saving to /tmp directory or use a smaller model. Error: z!Unsloth: GGUF conversion failed: r Modelfilerrdrz,Warning: Could not create Ollama modelfile: zuUnsloth: ##### The current model auto adds a BOS token. Unsloth: ##### We removed it in GGUF's chat template for you.r)z>Unsloth: example usage for Multimodal LLMs: llama-mtmd-cli -m z --mmproj zBUnsloth: load image inside llama.cpp runner: /image test_image.jpgz+Unsloth: Prompt model to describe the imagez=Unsloth: example usage for text only LLMs: llama-cli --model z -p "why is the sky blue?"z#Unsloth: Saved Ollama Modelfile to zuUnsloth: convert model to ollama format by running - ollama create model_name -f ./Modelfile - inside save directory.z4Unsloth: Saved Ollama Modelfile to current directoryzxUnsloth: convert model to ollama format by running - ollama create model_name -f ./Modelfile - inside current directory.)r gguf_filesmodelfile_locationrrr rI)1rrrorpr#rXanyrKrrrrrrrrQunsloth_generic_saverrrrrrrr is_availablerrrrrrrrlrrrrr&rr rYrZr[getcwdrrBrr))rrrrrrrrrrrrrrrrrcrrrrr[rrr old_chat_templaterrrrrquantization_methodsrrall_file_locationsrrr^ollama_successrrs) rKunsloth_save_pretrained_ggufrgslfIJJ4()B)BSXY$**3/3  n  FtX74;;#H#H [[..  @74;;@CjNCL DKK 1 1 ))-@@ DKK . . &&*@@ VXIIg&Ik$Im#* - #1 - &'($%(,,#$.!+B9CVCV+W( (+B9+M( ( , W,QQ[\?)y) '' "3 1X :: " " $ $ JJ " " $  ' 4 [[++   #)+{j/H HH EMM )#K ENN *$KRS S 12& )4 0 0  +S 1 1## +U 3 3"&':"; X  ))<=OA|'--/L.$ !11% ,' %% ' ' (:(:(< = >HAM##%$,"6/# B >/.N F3IPST 30B10E $ )+nk)R&)+biik;)O&,cgF$JJy)G!%  L  d  LM_`aMbLccmoABDoEnF G  RS ;< KL^_`LaKbb| } - 34F3GHI  D m DE  G )(02&  a4++33$**3/3 @ ?9!=>>?<   4QC7OPQ  b  H  !B1#FG G H6GF F @D E E Fs2U! V 8A V-V-V-2 V- W.BX!<X X!,V  V*V%%V*- W7WW X  'XX  XX!! Y+X??Ycb Uc [S5eSU;aURS5SOUnU(dUcSSKnURSS9nUnSnOUnS n[ S 5 [ UUUUUS SU U UUS 9 nUS nUS nUSnUSnUSnUSn[ S5 SSK J n U"US9n SU;aU R5Sn!U!SU3n"OUn"U RU"SUSS9 UHn#[RR!U#5n$U(a+SU$;a%SU$;aU$RSS5SOU$n%USU%3n&O/U$R#[RR!U5U5n&[ SU&S35 U R%U#U&U"SUU U U S9 M [RR'US5n'[RR)U'5(a#[ S5 U R%U'SU"SUS 3U U S!9 U(aG[RR)U5(a#[ S"5 U R%US#U"SUS$3U U S!9 S%U(aS&OS'S(URS5SS)US*US+3 n(UHn)[RR!U)5n$U(a+SU$;a%SU$;aU$RSS5SOU$n%USU%3n&O/U$R#[RR!U5U5n&U(S,U&S-3- n(M U(a+U(a$U(S.- n(U(S/- n(U(S0- n(U(S1- n(U(S2- n(U(S3- n(U(S4- n(OU(a U(S5- n(U(S6- n(U(a U(S7- n(U(S8- n(U(S9- n([RR'US:5n*[+U*S;5n+U+R-U(5 SSS5 U R%U*S:U"SSQ5 U(aUR1S?5 U R3U"USS@9 U(aH[RR)U5(a$[ SB5 SSKnUR"U5 U"$U"$![ a9nU(aSSKnUR"U5 O! O=f[SU35eSnAff=f!,(df  GN =f! N=f![ an[SAU35eSnAff=f! U"$=f!U(aO[RR)U5(a*[ SB5 SSKnUR"U5 f! f=fff=f)Cad Same as .push_to_hub(...) except 4bit weights are auto converted to float16 then converted to GGUF / llama.cpp format. Choose for `quantization_method` to be: "not_quantized" : "Recommended. Fast conversion. Slow inference, big files.", "fast_quantized" : "Recommended. Fast conversion. OK inference, OK file size.", "quantized" : "Recommended. Slow conversion. Fast inference, small files.", "f32" : "Not recommended. Retains 100% accuracy, but super slow and memory hungry.", "f16" : "Fastest conversion + retains 100% accuracy. Slow and memory hungry.", "q8_0" : "Fast conversion. High resource use, but generally acceptable.", "q4_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K", "q5_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K", "q2_k" : "Uses Q4_K for the attention.vw and feed_forward.w2 tensors, Q2_K for the other tensors.", "q3_k_l" : "Uses Q5_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_K", "q3_k_m" : "Uses Q4_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_K", "q3_k_s" : "Uses Q3_K for all tensors", "q4_0" : "Original quant method, 4-bit.", "q4_1" : "Higher accuracy than q4_0 but not as high as q5_0. However has quicker inference than q5 models.", "q4_k_s" : "Uses Q4_K for all tensors", "q5_0" : "Higher accuracy, higher resource usage and slower inference.", "q5_1" : "Even higher accuracy, resource usage and slower inference.", "q5_k_s" : "Uses Q5_K for all tensors", "q6_k" : "Uses Q8_K for all tensors", NrIrrr unsloth_gguf_)prefixTFz+Unsloth: Converting model to GGUF format...) rrrrrrrrrrcrr]r^rrr rz!Failed to convert model to GGUF: z-Unsloth: Uploading GGUF to Huggingface Hub...rrrrb)rnrrrrrz Uploading r)rrrnrrrrrrzUploading config.json...z - config)rrrnrrrrzUploading Ollama Modelfile...r\z - Ollama Modelfilez'--- tags: - gguf - llama.cpp - unsloth z- vision-language-modelrz --- # z : GGUF This model was finetuned and converted to GGUF format using [Unsloth](https://github.com/unslothai/unsloth). **Example usage**: - For text only LLMs: `./llama.cpp/llama-cli -hf zC --jinja` - For multimodal models: `./llama.cpp/llama-mtmd-cli -hf z% --jinja` ## Available Model files: z- `z` u) ## ⚠️ Ollama Note for Vision Models z[**Important:** Ollama currently does not support separate mmproj files for vision models. z2To create an Ollama model from this vision model: zR1. Place the `Modelfile` in the same directory as the finetuned bf16 merged model z23. Run: `ollama create model_name -f ./Modelfile` z2 (Replace `model_name` with your desired name) z;This will create a unified bf16 model that Ollama can use. z ## Ollama z5An Ollama Modelfile is included for easy deployment. z ## Note zDThe model's BOS token behavior was adjusted for GGUF compatibility. zThis was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) [](https://github.com/unslothai/unsloth) z README.mdrz Add READMEz>Unsloth: Successfully uploaded GGUF to https://huggingface.co/)ggufz llama-cpprzvision-language-model)rnrrz&Failed to upload to Hugging Face Hub: z'Unsloth: Cleaning up temporary files...)rr#tempfilemkdtemprQrgrrGrarrkrrrrYrZbasenamerrr[r]rrBextendradd_tags),rrnrrrrrrrrrrrrrrcrrrltemp_dirr cleanup_tempresultrer^rrr actual_save_directoryrrGrapir& full_repo_idrf original_name quant_suffix proper_name config_pathreadme_contentr readme_pathrEs, rKunsloth_push_to_hub_ggufr} s"XIJJ,/'>s#B'wJ|+##_#=! #  79 D-+!"5/+!3!3#7  $L1#$89$%:;!/ &'7 8 9:x)E" g zz|F+H&Zq 2L"L "  0MGG,,];M= @m+"''Q/2& ", Al^< +33GG$$^4j  J{m3/ 0 OO"/*&#!/%7%#  %0<ggll#8-H 77>>+ & & , - OO"-,&#$2#39!=%#   "''..1C"D"D 1 2 OO"4*&#$2#33F!G%#   %"-. ==b5 6=I>::AC$'DGG,,T2M= @m+"''Q/2& ", Al^< +33GG$$^4j  K=4 4N'" ( K KN } }N S SN s sN S SN T TN N N o -N V VN  m +N W N  r  ggll#8+F +s #q GGN #$ )&")!   L\N [ <D 45  KK/ 0  LL&#   BGGNN>:: ; <   n- <O D    n- >qcBCCDp$ #<   ICA3GHHI   BGGNN>:: ; <   n- ;Converted LoRA to GGML and uploaded to https://huggingface.co/ This GGML making function was made by Maheswar. Ping him @Maheswar on the Unsloth Discord or on HuggingFace (@mahiatlinux) if you like this!)rYrZr]rr`r7rrKr[rrrrrorr[rQrErFrirBrCrkrlrr)rrrnrrrrrrrrcrpython_installmakefile git_clonerlora_directory_pushr output_filerprqrrrr&links rK,unsloth_convert_lora_to_ggml_and_push_to_hubr= sW 77>>+ & & 8*FN    ! &% 8H<>I8*FN NN :I !I IIIJ0J  Jc [RRS5(dt[(a([ S/5nUR 5 [ SS9 SnOC[5n[ S/5nUR 5 [5nUR 5 OSn[S5Hn[R"5 M [XU5 URRn [RRUS5n [!SUS35 [!S U 35 S US U S 3n ["R$"U S ["R&["R&SS S9n U R(Hn [!U SS S9 M U R*Hn [!U SS S9 M U R 5 U R,S:wa!["R."U R,U 5eSSS5 [!S5 [!SU 35 [!S5 g!,(df  N3=f!["R.a"n[!SUR,35 SnAgSnAff=f)NrPrFrrrrrrrrrTrrrrrrrrr)rYrZr]rr`r7rrKr[rrrrrorr[rQrErFrirBrCrkrl)rrrrcrrrrrrrrprqrrrs rK-unsloth_convert_lora_to_ggml_and_save_locallyr s 77>>+ & & 8*FN    ! &% 8H<>I8*FN NN ::JJZ[ $[M 23:>:J!K=X^_G    ____!%   d"d3" d"d3" GGI}}! 33BMM7KK" $  8 FG  Y)    ( ( :1<<.IJs7+5H A>H H HHHI1II)merge_and_overwrite_loraprepare_saving)r rc Uc Ub [5nUbUc [S5e[RR [RR SS55(d [ SS9 /nUb[U[5(aOE[U[5(aU/nO,[U[5(a [U5nO [S5e[U5HTupU R5n U S:XaSn OU S :XaS n OU S :XaS n OU cS n URU R55 MV OURUR55 UHUn U [R!5;dMS U S3n [R#5HupU SU SU S3- n M [U 5e UH@n[%USUS9n UcM['UUSSSUS9 SSKJn U"US9nUR-UUSS/S9 MB W $)N.Unsloth: Please specify a token for uploading!rPzunsloth_convert_hf_to_gguf.pyT)just_clone_reporr.r3r/r4r0r5rrrrr))rrr)rrrrrrrrbz*.gguf)rrnrallow_patterns)r rrYrZr]r[r rrlrrrrrrrOrrP_convert_to_ggufrrkrr")rbrrrrrnrrrrrrRrSmetadatarrus rKsave_to_gguf_genericr s }, u}KLL 77>>"'',,{4ST U UD1 "& )4 0 0  +S 1 1## +U 3 3"&':"; X  ))<=OA'--/L.$ !11% ,' %% $ + +L,>,>,@ A > !''(9(?(?(AB0 ~224 4/ ~=bcE,224 1SEugR005u% % 16#  1    "!'   .&C   ,!#"*  '62 OrJunsloth_finetuned_mergecUcU(a [5nUS:Xa [S5eUS:XaSn[[UUUUUUUSSSS9 g)NrrrTF) rbrrrrrr output_dtypelow_disk_space_usage use_temp_file)r rrr)rbrrrrrrrrrrrrrrrrrrrrcrs rKr`r`3 sk6 } m# P  , ,# '!!#  rJcUc[R"S5 [[55nUUS'US [ S0UD6 [ S5Hn[ R"5 M g)rNrrbrrrIrrtrrr`rrrrs rK&unsloth_generic_save_pretrained_mergedrk s_6 L VXIIg&%9% 1X rJcUc[R"S5 [[55nUUS'UUS'SUS'US US [ S 0UD6 [ S 5Hn[ R"5 M grrrs rK"unsloth_generic_push_to_hub_mergedr s|6 H VXIIg")I#Im&)%9% 1X rJc [U5 [U[5(a#[UUUURR UUS9 gSnU(a UR XUS9 UR XS9 gURXS9 URU5 g)zXSave a QAT-trained model by converting fake-quantized weights to real quantized weights.rbrrtorchao_configrrNFrrrr)rrr"'_unsloth_save_torchao_with_given_configrorrr_)rbrrrrrs rK*_unsloth_save_torchao_with_attached_configr s5!%-../+!"\\==%     U   n< nV!!.1rJcU(a UcS5eUcS5e[[55nSUS'SUS'US [U[5(d8[U[5(d#UR U5 UR U5 O [ S0UD6 [S 5Hn[R"5 M S S K J nJ n J n Jn Jn S S KJn [X:5(aUnOU "US 9nSn[%US5(a`[%UR&S5(aE[)SUR&R*55nU=(d [%UR&S5nU(aU OUnU(aU OU nUR-U5n[.(aS[0R20nOS[0R20nUR,"U4SUS.UD6nUS-n[5[7S55[5S5:nSnU(a"UR9UUUS9 UR9UUS9 O!UR UUS9 UR U5 [:R<R?U5(a[@RB"U5 gg! g=f)aQuantizes the model with torchao and saves a torchao quantized checkpoint Args `save_directory`: local folder path or huggingface hub ID when `push_to_hub` is set to True, e.g. `my_model` `torchao_config` (TorchAOBaseConfig): configuration for torchao quantization, full list: https://docs.pytorch.org/ao/main/api_ref_quantization.html#inference-apis-for-quantize `push_to_hub` (bool): whether to push the checkpoint to huggingface hub or save locally NrzHUnsloth: Please specify a torchao_config for post-training quantization!Frrrrrr)AutoModelForCausalLM AutoTokenizer TorchAoConfigAutoModelForImageTextToText AutoProcessor) quantize_)rrorKc3D# UHnURS5v M g7frMrPrRs rKrT:_unsloth_save_torchao_with_given_config.. s$ / JJI J J/rVrWr1rauto) device_maprz-torchaotorchaoz0.14.0rrrrI)"rrrr"r#r_r`rrr transformersrrrrrrrrXror_rKfrom_pretrainedrrrrimportlib_versionrrYrZr]rGra)rbrrrrrrrrrrrrrrr auto_modelauto_processorkwargsquantized_modeltorchao_save_directoryrs rKrr sq  R"RR  "RQR"VXI$Im-Im"# e1 2 2:eY;W;W n-!!.1)y) 1X ".00,+HFuhGELL/$J$J \\//  A75<<A06,I05>>*!001  O,j8!!29!=>ARR## "9KUZ $  4eD'' "9K (  !!"89 ww~~n%%  MM. )&  s I66I:c 2UcU(a [5n[US5=(a URSLnUbU(aS5e[UUUUUUS9 OU(dS5e[ UUUUUS9 [ S5Hn[ R"5 M g)aSaves a torchao quantized model checkpoint. This function handles two mutually exclusive workflows: 1. **QAT (Quantization-Aware Training)**: If the model was trained with `qat_scheme` parameter, do NOT pass `torchao_config`. The function will convert the QAT fake-quantized weights to real quantized weights and save directly. 2. **PTQ (Post-Training Quantization)**: If you want to apply quantization to a regular model, pass a `torchao_config`. The model must NOT have been trained with `qat_scheme`. Args: `save_directory`: local folder path or huggingface hub ID when `push_to_hub` is True `tokenizer`: the tokenizer to save alongside the model `torchao_config` (TorchAOBaseConfig): configuration for torchao quantization. Required for PTQ, must be None for QAT models. Options: https://docs.pytorch.org/ao/main/api_ref_quantization.html#inference-apis-for-quantize `push_to_hub` (bool): whether to push to huggingface hub or save locally `token`: HuggingFace token for pushing to hub N_torchao_configzUnsloth: You passed `torchao_config` but this model was trained with `qat_scheme`. For QAT models, do not pass `torchao_config` - the quantization config is already attached to the model from training.rzUnsloth: No `torchao_config` provided and model was not trained with `qat_scheme`. Either train with `qat_scheme` parameter, or provide a `torchao_config` for post-training quantization.)rbrrrrr)r rXrrrrrr)rrrrrrhas_qat_configrs rKunsloth_save_pretrained_torchaorK s: }  '(MT-A-A-M!!  3 ! 0+!+%    * ~ 3+!%  1X rJc[S5e)NzAUnsloth: Sorry GGUF is currently not supported for vision models!)NotImplementedError)rmrs rKnot_implemented_saver s K rJcSSKnSSKnSSKJnJnJnJn URRS:Xa URnO URn[URU55RSS5n U SSn [R"SSU 5n URR!S 5R#S 5n S U S U S 3n [%U ['55 Un [)U S 5(aiU RRS:waOU RU l UR+[,U 5U l[)U S5(aU R/S/5 [)U S5(a U R0n OOMU(d[)US5(aUR+[2U5UlUR+[6U5UlUR+[:U5UlUR+[>U5Ul UR+[BU5Ul"UR+[FU5Ul$UR+[JU5Ul&U$UR+[2U5UlUR+[6U5UlUR+[:U5UlUR+[>U5Ul UR+[BU5Ul"U$)Nr)rrrrunsloth_push_to_hubNoneTyperrzz torch.saverdzdef unsloth_push_to_hub(self, z: """ a """ arguments = dict(locals()) del arguments["self"] if "tags" in arguments and arguments["tags"] is not None: assert(isinstance(arguments["tags"], (list, tuple))) arguments["tags"] = list(arguments["tags"]) + ["unsloth",] elif "tags" in arguments: arguments["tags"] = ["unsloth",] elif hasattr(self, "add_model_tags"): self.add_model_tags(["unsloth",]) if "commit_message" in arguments: commit_message = arguments["commit_message"] if commit_message is not None: if not commit_message.endswith(" "): commit_message += " " if "Unsloth" not in commit_message: commit_message += "(Trained with Unsloth)" else: commit_message = "Upload model trained with Unsloth" arguments["commit_message"] = commit_message if "commit_description" in arguments: commit_description = arguments["commit_description"] if commit_description is not None: if not commit_description.endswith(" "): commit_description += " " if "Unsloth" not in commit_description: commit_description += "(Trained with Unsloth 2x faster)" else: commit_description = "Upload model trained with Unsloth 2x faster" arguments["commit_description"] = commit_description # Update model tag if hasattr(self, "config"): _ = upload_to_huggingface( self, arguments["repo_id"], arguments["token"], "finetuned", "trl", file_location = None, old_username = None, private = arguments["private"], ) pass try: self.original_push_to_hub(**arguments) except: del arguments["tags"] self.original_push_to_hub(**arguments) pass if hasattr(self, "config"): print("Saved model to https://huggingface.co/" + arguments["repo_id"]) pass rrrrbro)'inspecttypestypingrrrrr__name__rr signaturerrr __doc__encoderjexecglobalsrX MethodTyperrrbrpush_to_hub_mergedrsave_pretrained_mergedr}push_to_hub_ggufrgsave_pretrained_ggufrsave_pretrained_torchaorpush_to_hub_ggmlrsave_pretrained_ggml) rbvisionrrrrrrrrdocspush_to_hub_textr;s rKr'r' s66 !!%::$99$00G%%&:;<DDZQWXI!" I/yII  ' ' . .w 7 > >w GD9)E F3 5l 79%N  NM 2 2**337LL2@2L2LN /).)9)9#^*N &~'788--! >7 + ++11N ) .  5( # #','7'72E(E $,1+;+;6,E (&+%5%56NPU%VE ").)9)9,e*E &-2,<,</-E )&+%5%5rFrgr}rrrunsloth_zoo.saving_utilsrrrrr`rrrrrrr'rIrJrKrs &;C9:222  ;KK ;3/1V'!;)0  $))BJJOO-. .!X-#x/    OQM V   \   [  K j j e d d ) + n  )!" #$  J H) '05<,"* 1C ' !>*&,@   (, !%#jj&+#!!#'$:"K@"%1n $#r{{*+n $ n $  n $ E#t)$ % n $n $n $n $#s(On $n $c]n $n $4.n $ SM!n $"d^#n $$%n $&'n $()n $* s)+n $./n $0 1n $n $b )#X,.>C L> "#4* H<H<H<H< H<  H<H< H<H<\%(, !%#jj&+#!!@"&!&#r{{*+& &  & E#t)$ % &&&&#s(O&&c]&& s)&&  !&X%#'$:"$(,1#K $@"&!) ) ) 4. ) SM )d^) sD !)#sD.)))))) 49 ))  !)X 2$.  $"X Xv(B5p'.#'.3'.S'.RU'.T!0!0#!0C!0H+s++RU+0* (," !%#jj&+#!!@"&%P#r{{*+P  P  P E#t)$ %Pd^PPPP#s(OPPc]PP s)!P"#P$ %Pl* #'$:"$(,1#K $@"&#^ ^  ^ 4. ^SM^d^^ sD !^#sD.)^^^^^ 49 ^ !^" #^D &$($I"$(I@"&SS4. S SM S d^ S sD !SSSSS StA"& == =  =@0   QQh/H(, !%#jj&+#!!#'$:"K@"%14 #r{{*+4  4  4 E#t)$ % 4 4 4 4 #s(O4 4 c]4 4 4.4 SM!4 "d^#4 $%4 &'4 ()4 * s)+4 ./4 0 14 4 t%(, !%#jj&+#!!@"&!&#r{{*+& &  & E#t)$ % &&&&#s(O&&c]&& s)&&  !&X%#'$:"$(,1#K $@"&!) ) ) 4. ) SM )d^) sD !)#sD.)))))) 49 ))  !)`(, 2#r{{*+ 2 2 E#t)$ % 2P(, e#r{{*+e  e E#t)$ % eV(, C#r{{*+C  C E#t)$ % CL EG[;;33;::s$(W--X0W99XXX