Type Alias: GgufMetadataTokenizer

type GgufMetadataTokenizer: {
  ggml: {
     model:   | "no_vocab"
        | "llama"
        | "gpt2"
        | "bert"
        | string;
     pre:   | "default"
        | "llama3"
        | "llama-v3"
        | "llama-bpe"
        | "deepseek-llm"
        | "deepseek-coder"
        | "falcon"
        | "mpt"
        | "starcoder"
        | "gpt-2"
        | "jina-es"
        | "jina-de"
        | "jina-v2-es"
        | "jina-v2-de"
        | "refact"
        | "command-r"
        | "qwen2"
        | "stablelm2"
        | "olmo"
        | "dbrx"
        | "smaug-bpe"
        | string;
     tokens: readonly string[];
     token_type: GgufMetadataTokenizerTokenType[];
     token_type_count: number;
     scores: readonly number[];
     merges: readonly string[];
     bos_token_id: number;
     eos_token_id: number;
     unknown_token_id: number;
     separator_token_id: number;
     padding_token_id: number;
     add_bos_token: boolean;
     add_eos_token: boolean;
     add_space_prefix: boolean;
     added_tokens: readonly string[];
     prefix_token_id: number;
     suffix_token_id: number;
     middle_token_id: number;
     eot_token_id: number;
    };
  huggingface: {
     json: string;
    };
  chat_template: string;
};

Type declaration

ggml

readonly ggml: {
  model:   | "no_vocab"
     | "llama"
     | "gpt2"
     | "bert"
     | string;
  pre:   | "default"
     | "llama3"
     | "llama-v3"
     | "llama-bpe"
     | "deepseek-llm"
     | "deepseek-coder"
     | "falcon"
     | "mpt"
     | "starcoder"
     | "gpt-2"
     | "jina-es"
     | "jina-de"
     | "jina-v2-es"
     | "jina-v2-de"
     | "refact"
     | "command-r"
     | "qwen2"
     | "stablelm2"
     | "olmo"
     | "dbrx"
     | "smaug-bpe"
     | string;
  tokens: readonly string[];
  token_type: GgufMetadataTokenizerTokenType[];
  token_type_count: number;
  scores: readonly number[];
  merges: readonly string[];
  bos_token_id: number;
  eos_token_id: number;
  unknown_token_id: number;
  separator_token_id: number;
  padding_token_id: number;
  add_bos_token: boolean;
  add_eos_token: boolean;
  add_space_prefix: boolean;
  added_tokens: readonly string[];
  prefix_token_id: number;
  suffix_token_id: number;
  middle_token_id: number;
  eot_token_id: number;
};

ggml.model

readonly model: 
  | "no_vocab"
  | "llama"
  | "gpt2"
  | "bert"
  | string;

ggml.pre?

readonly optional pre: 
  | "default"
  | "llama3"
  | "llama-v3"
  | "llama-bpe"
  | "deepseek-llm"
  | "deepseek-coder"
  | "falcon"
  | "mpt"
  | "starcoder"
  | "gpt-2"
  | "jina-es"
  | "jina-de"
  | "jina-v2-es"
  | "jina-v2-de"
  | "refact"
  | "command-r"
  | "qwen2"
  | "stablelm2"
  | "olmo"
  | "dbrx"
  | "smaug-bpe"
  | string;

ggml.tokens

readonly tokens: readonly string[];

ggml.token_type

readonly token_type: GgufMetadataTokenizerTokenType[];

ggml.token_type_count?

readonly optional token_type_count: number;

ggml.scores?

readonly optional scores: readonly number[];

ggml.merges?

readonly optional merges: readonly string[];

ggml.bos_token_id?

readonly optional bos_token_id: number;

ggml.eos_token_id?

readonly optional eos_token_id: number;

ggml.unknown_token_id?

readonly optional unknown_token_id: number;

ggml.separator_token_id?

readonly optional separator_token_id: number;

ggml.padding_token_id?

readonly optional padding_token_id: number;

ggml.add_bos_token?

readonly optional add_bos_token: boolean;

ggml.add_eos_token?

readonly optional add_eos_token: boolean;

ggml.add_space_prefix?

readonly optional add_space_prefix: boolean;

ggml.added_tokens?

readonly optional added_tokens: readonly string[];

ggml.prefix_token_id?

readonly optional prefix_token_id: number;

ggml.suffix_token_id?

readonly optional suffix_token_id: number;

ggml.middle_token_id?

readonly optional middle_token_id: number;

ggml.eot_token_id?

readonly optional eot_token_id: number;

huggingface?

readonly optional huggingface: {
  json: string;
};

huggingface.json?

readonly optional json: string;

chat_template?

readonly optional chat_template: string;

Defined in

gguf/types/GgufMetadataTypes.ts:220