
    >Ci'&                    "   S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKJr  S SK	J	r	  S SK
JrJrJr  S\R                  ;  a  \" \5      R                   R                   R                   R                   S-  R#                  5       (       aF  \R$                  R'                  S \" \" \5      R                   R                   R                   5      5        S SKr\R,                  " S5      r " S	 S
\5      rSS jrSS jrSS jrSS jr\S:X  a  \" 5         gg)    )annotationsN)Path)tqdm)AnySequence
NamedTupleNO_LOCAL_GGUFzgguf-pyzgguf-new-metadatac                  F    \ rS rSr% S\S'   S\S'   SrS\S'   S	rS
\S'   Srg	)MetadataDetails   zgguf.GGUFValueTypetyper   value strdescriptionNzgguf.GGUFValueType | Nonesub_type )__name__
__module____qualname____firstlineno____annotations__r   r   __static_attributes__r       X/home/james-whalen/.local/lib/python3.13/site-packages/gguf/scripts/gguf_new_metadata.pyr   r      s"    
JK*.H'.r   r   c                V    U R                  U5      nU(       a  UR                  5       $ S $ N)	get_fieldcontents)readerkeyfields      r   get_field_datar#      s&    S!E$5>>.$.r   c                    [        U 5       VVs/ s H  u  p#X1:X  d  M  UPM     nnn[        U5      S:X  a  [        SU S35      eU$ s  snnf )Nr   zUnable to find "z" in token list!)	enumeratelenLookupError)
token_listtokenindexr   	token_idss        r   
find_tokenr,   $   sQ    +4Z+@S+@<5EN+@IS
9~,UG3CDEE Ts
   AAc                	   U R                   R                  5        GH  nUR                  [        R                  R
                  R                  :X  d   UR                  R                  S5      (       a$  [        R                  SUR                   35        Mz  UR                  R                  [        R                  R                  R                  5      (       aM  [        R                  R                  R                  U;   a%  [        R                  SUR                   35        GM	  UR                  U;   a%  [        R                  SUR                   35        GM>  UR                  S   nU[        R                  R                  :X  a  UR                  S   OS n[        XTR!                  5       US9nUR#                  UR                  U5      nUR                  U;   aV  [        R                  SUR                   S	UR$                   S
UR$                   SUR&                   35        X$R                  	 O/UR$                  b"  [        R                  SUR                   35        UR$                  c  GMU  UR)                  UR                  UR$                  UR*                  UR,                  c  UOUR,                  S9  GM     [        R                  R                  R                  U;   az  [        R                  S5        UR/                  U[        R                  R                  R                     R$                  5        U[        R                  R                  R                  	 UR1                  5        H]  u  p[        R                  SU	 S	UR$                   SUR&                   35        UR)                  XR$                  UR*                  5        M_     Sn
U R2                   Hv  nXR4                  -  n
UR7                  UR                  UR8                  R:                  UR8                  R<                  UR8                  R>                  UR@                  5        Mx     [C        SU
SSS9nURE                  5         URG                  5         URI                  5         U R2                   H9  nURK                  UR8                  5        URM                  UR4                  5        M;     URO                  5         g )NzGGUF.zSuppressing z	Skipping z	Removing r   )r   z
Modifying z: "z" -> "z" zCopying zAdding chat template(s)zAdding WritingbyteT)desctotalunit
unit_scale)(fieldsvaluesnameggufKeysGeneralARCHITECTURE
startswithloggerdebug	TokenizerCHAT_TEMPLATEtypesGGUFValueTypeARRAYr   r   getr   r   add_key_valuer   r   add_chat_templateitemstensorsn_bytesadd_tensor_infodatashapedtypenbytestensor_typer   write_header_to_filewrite_kv_data_to_filewrite_ti_data_to_filewrite_tensor_dataupdateclose)r    writernew_metadataremove_metadatar"   val_typer   old_valvalr!   total_bytestensorbars                r   copy_with_new_metadatar_   -   s   %%'::**7775::;P;PQX;Y;YLL<

|45 ::  !4!4!B!BCC		H[H[HiHimyHyLL9UZZL12::(LL9UZZL12;;q>&.$2D2D2J2J&J5;;r?PT!(NN,<xPuzz73::%LL:ejj\W]]O6#))TVWZWfWfVghiZZ(YY"LL8EJJ<0199   SYYWZWcWcWk8qtq}q} ~7 (: yy((L8./  dii.A.A.O.O!P!V!VW,,::; &&(wse3syykCOO3DEFS))SXX6 ) K..~~%v{{FKK,=,=v{{?P?PRXR]R]RdRdflfxfxy ! I[v$
OC
!
  "
  "..  -

6>>" ! LLNr   c                    S [         R                  R                  R                  R	                  5        5       n [        S U  5       5      n[        R                  " SS9nUR                  S[        SS9  UR                  S[        S	S9  UR                  S
[        SSS9  UR                  S[        SSS9  UR                  S[        SSS9  UR                  S[        SSS9  UR                  S[        SSS9  UR                  SS[        SSS9  UR                  SS[        S S!S"R                  UR	                  5       5      S#4S$9  UR                  S%S[        S&S!S"R                  UR	                  5       5      S'4S$9  UR                  S(S)S*S+9  UR                  S,S)S-S+9  UR                  [        [        R                  5      S!:  a  S OS./5      n[         R"                  " UR$                  (       a  [         R&                  O[         R(                  S/9  0 nUR*                  =(       d    / nUR,                  (       aS  [/        [         R0                  R2                  UR,                  5      U[         R                  R4                  R6                  '   UR8                  (       aS  [/        [         R0                  R2                  UR8                  5      U[         R                  R4                  R:                  '   UR<                  (       a  [/        [         R0                  R2                  UR<                  R?                  S05      (       a   [@        RB                  " UR<                  5      OUR<                  5      U[         R                  R                  RD                  '   URF                  (       a  [I        URF                  S15       n[@        RJ                  " U5      nURM                  S25      nU(       aI  [/        [         R0                  R2                  U5      U[         R                  R                  RD                  '   S S S 5        URN                  (       aS  [/        [         R0                  R2                  URN                  5      U[         R                  R                  RP                  '   U(       a  [R        RU                  S35        [R        RU                  S45        [R        RU                  S55        URV                  (       dQ  [R        RU                  S65        [Y        S75      n	U	S8:w  a+  [R        R[                  S95        [        R\                  " S:5        [R        R[                  S;URX                   35        [         R^                  " URX                  S15      n
[a        U
[         R                  R4                  Rb                  5      n[a        U
[         R                  R                  Rd                  5      =(       d    / nURf                  =(       d    /  H  u  pX;  a  [R        RU                  S<U S=35        M%  [i        X5      n[/        [         R0                  Rj                  US:   S>U 35      XAU   '   [        U5      S?:  d  Mq  [R        RU                  S@U SAUS:    SB35        [R        RU                  SCR                  SD U 5       5      5        M     URl                  =(       d    /  H  u  nnX;  a  [R        RU                  S<U S=35        M&  URo                  5       (       d  [q        SEU SF35      e[s        U5      nUS::  aA  U[        U5      :  a2  [/        [         R0                  Rj                  US>UU    35      XAU   '   M  [q        SGU SH35      e   [t        Rv                  Ry                  URz                  5      (       a  URV                  (       d  [R        RU                  S35        [R        RU                  SIURz                   SJ35        [R        RU                  S65        [Y        S75      n	U	S8:w  a+  [R        R[                  S95        [        R\                  " S:5        [R        R[                  SKURz                   35        [         R|                  " URz                  XR~                  SL9n[a        U
[         R                  R4                  R                  5      nUb  [R        R                  SMU 35        UUlB        [        U
UXE5        g ! , (       d  f       GN= f)NNc              3     #    U  H@  oR                  S 5      (       a  M  [        [        R                  R                  U5      v   MB     g7f)_N)r<   getattrr8   r9   r?   .0ns     r   	<genexpr>main.<locals>.<genexpr>h   s8       ACfanznz{~n9'$))"5"5q99Cfs
   A
+A
c              3     #    U  H?  oR                  S 5      (       d  M  UR                  S5      S   S[        S 5      *  U4v   MA     g7f)	_token_id.r.   N)endswithsplitr&   rd   s     r   rg   rh   i   sC     xJ\Q`j`jkv`w@R();3{+;*;<a@J\s
   A	*A	z,Make a copy of a GGUF file with new metadata)r   inputz GGUF format model input filename)r   helpoutputz!GGUF format model output filenamez--general-namezThe models general.namez"name")r   ro   metavarz--general-descriptionzThe models general.descriptionz"Description ..."z--chat-templatez:Chat template string (or JSON string containing templates)z"{% ... %} ..."z--chat-template-configz'Config file containing chat template(s)ztokenizer_config.jsonz--pre-tokenizerzThe models tokenizer.ggml.prez"pre tokenizer"z--remove-metadataappendz/Remove metadata (by key name) from output modelzgeneral.url)actionr   ro   rq   z--special-tokenzSpecial token by value   z | z	"<token>")rs   r   ro   nargsrq   z--special-token-by-idzSpecial token by id0z--force
store_truez$Bypass warnings without confirmation)rs   ro   z	--verbosezIncrease output verbosityz--help)level[rchat_templatez&*** Warning *** Warning *** Warning **z=* Most metadata is required for a fully functional GGUF file,z@* removing crucial metadata may result in a corrupt output file!z<* Enter exactly YES if you are positive you want to proceed:zYES, I am sure> YESz(You didn't enter YES. Okay then, see ya!r   z* Loading: zUnknown special token "z", ignoring...z=    z
Multiple "z" tokens found, choosing ID z0, use --special-token-by-id if you want another:z, c              3  8   #    U  H  n[        U5      v   M     g 7fr   )r   )re   is     r   rg   rh      s     (=AQs   z
Token ID "z" is not a valid ID!z	Token ID z is not within token list!z* The "z3" GGUF file already exists, it will be overwritten!z* Writing: )arch	endianesszSetting custom alignment: )Dr8   r9   r?   __dict__keysdictargparseArgumentParseradd_argumentr   r   join
parse_argsr&   sysargvloggingbasicConfigverboseDEBUGINFOrX   general_namer   rB   STRINGr:   NAMEgeneral_descriptionDESCRIPTIONr{   r<   jsonloadsr@   chat_template_configopenloadrD   pre_tokenizerPREr=   warningforcern   infoexit
GGUFReaderr#   r;   LISTspecial_tokenr,   UINT32special_token_by_id	isdecimalr'   intospathisfilerp   
GGUFWriterr   	ALIGNMENTr>   data_alignmentr_   )tokenizer_metadatatoken_namesparserargsrW   rX   fpconfigtemplateresponser    r   r(   r7   r)   ids	id_stringid_intrV   	alignments                       r   mainr   g   s    A499CVCVC_C_CdCdCf  AxJ\xxK$$1_`F
DWyz
DWz{
(CWp  {C  D
/CWw  BU  V
)C  XT  ^o  p
0D  XA  Kb  c
)CWv  AR  S
+C  XI  S`  a
)CWowx  DI  DN  DN  OZ  O_  O_  Oa  Db  do  Cp  q
/CWltu  AF  AK  AK  LW  L\  L\  L^  A_  ad  @e  f
	W}~
WrsS]Q%6TXJGDt||gmmNL**0bO/>t?Q?Q?X?XZ^ZkZk/lTYY&&++,6EdFXFXF_F_aeayay6zTYY&&223:I$J\J\JcJc  HL  HZ  HZ  He  He  fi  Hj  Hjeieoeopt  qC  qC  fD  pt  pB  pB  ;CTYY((667  $++S1RYYr]Fzz/2HBQRVRdRdRkRkmuBvTYY00>>?	 2 0?@R@R@Y@Y[_[m[m0nTYY((,,-?@VWYZzzNNYZ/0H5 FG
KK+djj\*+__TZZ-F&$))"3"3"@"@AD		(;(;(@(@AGRJ))/R/"NN4TF.IJZ/C.=d>P>P>W>WY\]^Y_cefkelam.nLT*+3x!|E72NsSTvh  WG   H  Ityy(=(==> 0  339r9i"NN4TF.IJ$$&&
9+5IJKK^F{vJ72A$BTBTB[B[]cgijtu{j|i}e~2./!IfX5O"PQQ : 
ww~~dkk""4::?@-`abUV+,uKKBCHHQK
KK+dkk]+,__T[[t?O?OPFvtyy'8'8'B'BCI1)=> )66<IK 21s   =A8c88
d__main__)r    gguf.GGUFReaderr!   r   returnr   )r(   Sequence[int]r)   r   r   r   )
r    r   rV   zgguf.GGUFWriterrW   zdict[str, MetadataDetails]rX   zSequence[str]r   None)r   r   )
__future__r   r   r   r   r   r   pathlibr   r   typingr   r   r   environ__file__parentexistsr   insertr   r8   	getLoggerr=   r   r#   r,   r_   r   r   r   r   r   <module>r      s    "   	 
    , , "**$$x.*?*?*F*F*M*M*T*TW`*`)h)h)j)jHHOOAs4>0077>>?@ 			.	//j //7tgJT zF r   