Class: LlamaModel

Defined in: evaluator/LlamaModel/LlamaModel.ts:225

Properties

tokenizer

readonly tokenizer: Tokenizer;

Defined in: evaluator/LlamaModel/LlamaModel.ts:255

---

onDispose

readonly onDispose: EventRelay<void>;

Defined in: evaluator/LlamaModel/LlamaModel.ts:256

Accessors

disposed

Get Signature

get disposed(): boolean;

Defined in: evaluator/LlamaModel/LlamaModel.ts:386

Returns

boolean

---

llama

Get Signature

get llama(): Llama;

Defined in: evaluator/LlamaModel/LlamaModel.ts:390

Returns

Llama

---

tokens

Get Signature

get tokens(): LlamaModelTokens;

Defined in: evaluator/LlamaModel/LlamaModel.ts:394

Returns

LlamaModelTokens

---

filename

Get Signature

get filename(): string | undefined;

Defined in: evaluator/LlamaModel/LlamaModel.ts:398

Returns

string | undefined

---

fileInfo

Get Signature

get fileInfo(): GgufFileInfo;

Defined in: evaluator/LlamaModel/LlamaModel.ts:402

Returns

GgufFileInfo

---

fileInsights

Get Signature

get fileInsights(): GgufInsights;

Defined in: evaluator/LlamaModel/LlamaModel.ts:406

Returns

GgufInsights

---

architecture

Get Signature

get architecture(): GgufArchitectureType;

Defined in: evaluator/LlamaModel/LlamaModel.ts:410

Returns

GgufArchitectureType

---

gpuLayers

Get Signature

get gpuLayers(): number;

Defined in: evaluator/LlamaModel/LlamaModel.ts:418

Number of layers offloaded to the GPU. If GPU support is disabled, this will always be 0.

Returns

number

---

useMmap

Get Signature

get useMmap(): boolean;

Defined in: evaluator/LlamaModel/LlamaModel.ts:428

Whether the model is loaded using mmap (memory-mapped file) or not.

When Direct I/O (setting the useDirectIo option to true) is used it'll override mmap and this value may be out of sync with the actual usage of mmap for the loading of this model instance.

Returns

boolean

---

lazyMode

Get Signature

get lazyMode(): boolean | "auto";

Defined in: evaluator/LlamaModel/LlamaModel.ts:432

Returns

boolean | "auto"

---

size

Get Signature

get size(): number;

Defined in: evaluator/LlamaModel/LlamaModel.ts:441

Total model size in memory in bytes.

When using mmap, actual memory usage may be higher than this value due to llama.cpp's performance optimizations.

Returns

number

---

flashAttentionSupported

Get Signature

get flashAttentionSupported(): boolean;

Defined in: evaluator/LlamaModel/LlamaModel.ts:447

Returns

boolean

---

defaultContextFlashAttention

Get Signature

get defaultContextFlashAttention(): boolean | "auto";

Defined in: evaluator/LlamaModel/LlamaModel.ts:451

Returns

boolean | "auto"

---

defaultContextSwaFullCache

Get Signature

get defaultContextSwaFullCache(): boolean;

Defined in: evaluator/LlamaModel/LlamaModel.ts:455

Returns

boolean

---

defaultContextKvCacheKeyType

Get Signature

get defaultContextKvCacheKeyType(): GgmlType;

Defined in: evaluator/LlamaModel/LlamaModel.ts:459

Returns

GgmlType

---

defaultContextKvCacheValueType

Get Signature

get defaultContextKvCacheValueType(): GgmlType;

Defined in: evaluator/LlamaModel/LlamaModel.ts:463

Returns

GgmlType

---

memoryUsage

Get Signature

get memoryUsage(): {
  ram: number;
  vram: number;
};

Defined in: evaluator/LlamaModel/LlamaModel.ts:468

Assumed memory footprint of the model in bytes

Returns

{
  ram: number;
  vram: number;
}

ram

ram: number;

vram

vram: number;

---

trainContextSize

Get Signature

get trainContextSize(): number;

Defined in: evaluator/LlamaModel/LlamaModel.ts:773

The context size the model was trained on

Returns

number

---

embeddingVectorSize

Get Signature

get embeddingVectorSize(): number;

Defined in: evaluator/LlamaModel/LlamaModel.ts:783

The size of an embedding vector the model can produce

Returns

number

---

vocabularyType

Get Signature

get vocabularyType(): LlamaVocabularyType;

Defined in: evaluator/LlamaModel/LlamaModel.ts:792

Returns

LlamaVocabularyType

Methods

dispose()

dispose(): Promise<void>;

Defined in: evaluator/LlamaModel/LlamaModel.ts:372

Returns

Promise<void>

---

tokenize()

Call Signature

tokenize(
   text: string, 
   specialTokens?: boolean, 
   options?: "trimLeadingSpace"): Token[];

Defined in: evaluator/LlamaModel/LlamaModel.ts:488

Transform text into tokens that can be fed to the model

Parameters

Parameter	Type	Description
text	string	the text to tokenize
specialTokens?	boolean	if set to true, text that correspond to special tokens will be tokenized to those tokens. For example, <s> will be tokenized to the BOS token if specialTokens is set to true, otherwise it will be tokenized to tokens that corresponds to the plaintext <s> string.
options?	"trimLeadingSpace"	additional options for tokenization. If set to "trimLeadingSpace", a leading space will be trimmed from the tokenized output if the output has an additional space at the beginning.

Returns

Token[]

Call Signature

tokenize(text: BuiltinSpecialTokenValue, specialTokens: "builtin"): Token[];

Defined in: evaluator/LlamaModel/LlamaModel.ts:489

Transform text into tokens that can be fed to the model

Parameters

Parameter	Type	Description
text	BuiltinSpecialTokenValue	the text to tokenize
specialTokens	"builtin"	if set to true, text that correspond to special tokens will be tokenized to those tokens. For example, <s> will be tokenized to the BOS token if specialTokens is set to true, otherwise it will be tokenized to tokens that corresponds to the plaintext <s> string.

Returns

Token[]

---

detokenize()

detokenize(
   tokens: readonly Token[], 
   specialTokens?: boolean, 
   lastTokens?: readonly Token[]): string;

Defined in: evaluator/LlamaModel/LlamaModel.ts:603

Transform tokens into text

Parameters

Parameter	Type	Default value	Description
tokens	readonly Token[]	undefined	the tokens to detokenize.
specialTokens?	boolean	false	if set to true, special tokens will be detokenized to their corresponding token text representation. Recommended for debugging purposes only. > Note: there may be additional spaces around special tokens that were not present in the original text - this is not a bug, this is how the tokenizer is supposed to work. Defaults to false.
lastTokens?	readonly Token[]	undefined	the last few tokens that preceded the tokens to detokenize. If provided, the last few tokens will be used to determine whether a space has to be added before the current tokens or not, and apply other detokenizer-specific heuristics to provide the correct text continuation to the existing tokens. Using it may have no effect with some models, but it is still recommended.

Returns

string

---

getTokenAttributes()

getTokenAttributes(token: Token): TokenAttributes;

Defined in: evaluator/LlamaModel/LlamaModel.ts:624

Parameters

Parameter	Type
token	Token

Returns

TokenAttributes

---

isSpecialToken()

isSpecialToken(token: Token | undefined): boolean;

Defined in: evaluator/LlamaModel/LlamaModel.ts:635

Check whether the given token is a special token (a control-type token or a token with no normal text representation)

Parameters

Parameter	Type
token	Token | undefined

Returns

boolean

---

iterateAllTokens()

iterateAllTokens(): Generator<Token, void, unknown>;

Defined in: evaluator/LlamaModel/LlamaModel.ts:650

Returns

Generator<Token, void, unknown>

---

isEogToken()

isEogToken(token: Token | undefined): boolean;

Defined in: evaluator/LlamaModel/LlamaModel.ts:663

Check whether the given token is an EOG (End Of Generation) token, like EOS or EOT.

Parameters

Parameter	Type
token	Token | undefined

Returns

boolean

---

createContext()

createContext(options?: LlamaContextOptions): Promise<LlamaContext>;

Defined in: evaluator/LlamaModel/LlamaModel.ts:670

Parameters

Parameter	Type
options	LlamaContextOptions

Returns

Promise<LlamaContext>

---

createEmbeddingContext()

createEmbeddingContext(options?: LlamaEmbeddingContextOptions): Promise<LlamaEmbeddingContext>;

Defined in: evaluator/LlamaModel/LlamaModel.ts:687

Parameters

Parameter	Type
options	LlamaEmbeddingContextOptions

Returns

Promise<LlamaEmbeddingContext>

See

Using Embedding tutorial

---

createRankingContext()

createRankingContext(options?: LlamaRankingContextOptions): Promise<LlamaRankingContext>;

Defined in: evaluator/LlamaModel/LlamaModel.ts:697

Parameters

Parameter	Type
options	LlamaRankingContextOptions

Returns

Promise<LlamaRankingContext>

See

Reranking Documents tutorial

---

createDecisionContext()

createDecisionContext(options?: LlamaDecisionContextOptions): Promise<LlamaDecisionContext>;

Defined in: evaluator/LlamaModel/LlamaModel.ts:707

Parameters

Parameter	Type
options	LlamaDecisionContextOptions

Returns

Promise<LlamaDecisionContext>

See

Using Structured Decisions tutorial

---

getWarnings()

getWarnings(): string[];

Defined in: evaluator/LlamaModel/LlamaModel.ts:719

Get warnings about the model file that would affect its usage.

These warnings include all the warnings generated by GgufInsights, but are more comprehensive.

Returns

string[]