Type Alias: GgufMetadataTokenizer

type GgufMetadataTokenizer = {
  ggml: {
     model: "no_vocab" | "llama" | "gpt2" | "bert" | string;
     pre:   | "default"
        | "llama3"
        | "llama-v3"
        | "llama-bpe"
        | "deepseek-llm"
        | "deepseek-coder"
        | "falcon"
        | "falcon3"
        | "mpt"
        | "starcoder"
        | "gpt-2"
        | "phi-2"
        | "jina-es"
        | "jina-de"
        | "jina-v1-en"
        | "jina-v2-es"
        | "jina-v2-de"
        | "jina-v2-code"
        | "refact"
        | "command-r"
        | "qwen2"
        | "stablelm2"
        | "olmo"
        | "dbrx"
        | "smaug-bpe"
        | "poro-chat"
        | "chatglm-bpe"
        | "viking"
        | "jais"
        | "tekken"
        | "smollm"
        | "codeshell"
        | "bloom"
        | "gpt3-finnish"
        | "exaone"
        | "chameleon"
        | "minerva-7b"
        | "megrez"
        | "gpt-4o"
        | string;
     tokens: readonly string[];
     token_type: GgufMetadataTokenizerTokenType[];
     token_type_count: number;
     scores: readonly number[];
     merges: readonly string[];
     bos_token_id: number;
     eos_token_id: number;
     eot_token_id: number;
     eom_token_id: number;
     unknown_token_id: number;
     separator_token_id: number;
     padding_token_id: number;
     cls_token_id: number;
     mask_token_id: number;
     add_bos_token: boolean;
     add_eos_token: boolean;
     add_space_prefix: boolean;
     added_tokens: readonly string[];
     fim_pre_token_id: number;
     fim_suf_token_id: number;
     fim_mid_token_id: number;
     fim_pad_token_id: number;
     fim_rep_token_id: number;
     fim_sep_token_id: number;
     prefix_token_id: number;
     suffix_token_id: number;
     middle_token_id: number;
    };
  huggingface: {
     json: string;
    };
  chat_template: string;
};

Defined in: gguf/types/GgufMetadataTypes.ts:224

Properties

ggml

readonly ggml: {
  model: "no_vocab" | "llama" | "gpt2" | "bert" | string;
  pre:   | "default"
     | "llama3"
     | "llama-v3"
     | "llama-bpe"
     | "deepseek-llm"
     | "deepseek-coder"
     | "falcon"
     | "falcon3"
     | "mpt"
     | "starcoder"
     | "gpt-2"
     | "phi-2"
     | "jina-es"
     | "jina-de"
     | "jina-v1-en"
     | "jina-v2-es"
     | "jina-v2-de"
     | "jina-v2-code"
     | "refact"
     | "command-r"
     | "qwen2"
     | "stablelm2"
     | "olmo"
     | "dbrx"
     | "smaug-bpe"
     | "poro-chat"
     | "chatglm-bpe"
     | "viking"
     | "jais"
     | "tekken"
     | "smollm"
     | "codeshell"
     | "bloom"
     | "gpt3-finnish"
     | "exaone"
     | "chameleon"
     | "minerva-7b"
     | "megrez"
     | "gpt-4o"
     | string;
  tokens: readonly string[];
  token_type: GgufMetadataTokenizerTokenType[];
  token_type_count: number;
  scores: readonly number[];
  merges: readonly string[];
  bos_token_id: number;
  eos_token_id: number;
  eot_token_id: number;
  eom_token_id: number;
  unknown_token_id: number;
  separator_token_id: number;
  padding_token_id: number;
  cls_token_id: number;
  mask_token_id: number;
  add_bos_token: boolean;
  add_eos_token: boolean;
  add_space_prefix: boolean;
  added_tokens: readonly string[];
  fim_pre_token_id: number;
  fim_suf_token_id: number;
  fim_mid_token_id: number;
  fim_pad_token_id: number;
  fim_rep_token_id: number;
  fim_sep_token_id: number;
  prefix_token_id: number;
  suffix_token_id: number;
  middle_token_id: number;
};

Defined in: gguf/types/GgufMetadataTypes.ts:225

model

readonly model: "no_vocab" | "llama" | "gpt2" | "bert" | string;

pre?

readonly optional pre: 
  | "default"
  | "llama3"
  | "llama-v3"
  | "llama-bpe"
  | "deepseek-llm"
  | "deepseek-coder"
  | "falcon"
  | "falcon3"
  | "mpt"
  | "starcoder"
  | "gpt-2"
  | "phi-2"
  | "jina-es"
  | "jina-de"
  | "jina-v1-en"
  | "jina-v2-es"
  | "jina-v2-de"
  | "jina-v2-code"
  | "refact"
  | "command-r"
  | "qwen2"
  | "stablelm2"
  | "olmo"
  | "dbrx"
  | "smaug-bpe"
  | "poro-chat"
  | "chatglm-bpe"
  | "viking"
  | "jais"
  | "tekken"
  | "smollm"
  | "codeshell"
  | "bloom"
  | "gpt3-finnish"
  | "exaone"
  | "chameleon"
  | "minerva-7b"
  | "megrez"
  | "gpt-4o"
  | string;

tokens

readonly tokens: readonly string[];

token_type

readonly token_type: GgufMetadataTokenizerTokenType[];

token_type_count?

readonly optional token_type_count: number;

scores?

readonly optional scores: readonly number[];

merges?

readonly optional merges: readonly string[];

bos_token_id?

readonly optional bos_token_id: number;

eos_token_id?

readonly optional eos_token_id: number;

eot_token_id?

readonly optional eot_token_id: number;

eom_token_id?

readonly optional eom_token_id: number;

unknown_token_id?

readonly optional unknown_token_id: number;

separator_token_id?

readonly optional separator_token_id: number;

padding_token_id?

readonly optional padding_token_id: number;

cls_token_id?

readonly optional cls_token_id: number;

mask_token_id?

readonly optional mask_token_id: number;

add_bos_token?

readonly optional add_bos_token: boolean;

add_eos_token?

readonly optional add_eos_token: boolean;

add_space_prefix?

readonly optional add_space_prefix: boolean;

added_tokens?

readonly optional added_tokens: readonly string[];

fim_pre_token_id?

readonly optional fim_pre_token_id: number;

fim_suf_token_id?

readonly optional fim_suf_token_id: number;

fim_mid_token_id?

readonly optional fim_mid_token_id: number;

fim_pad_token_id?

readonly optional fim_pad_token_id: number;

fim_rep_token_id?

readonly optional fim_rep_token_id: number;

fim_sep_token_id?

readonly optional fim_sep_token_id: number;

prefix_token_id?

readonly optional prefix_token_id: number;

Deprecated

suffix_token_id?

readonly optional suffix_token_id: number;

Deprecated

middle_token_id?

readonly optional middle_token_id: number;

Deprecated

---

huggingface?

readonly optional huggingface: {
  json: string;
};

Defined in: gguf/types/GgufMetadataTypes.ts:264

json?

readonly optional json: string;

---

chat_template?

readonly optional chat_template: string;

Defined in: gguf/types/GgufMetadataTypes.ts:267