Class: Llama

Defined in: bindings/Llama.ts:40

Properties

onDispose

readonly onDispose: EventRelay<void>;

Defined in: bindings/Llama.ts:81

Accessors

disposed

Get Signature

get disposed(): boolean;

Defined in: bindings/Llama.ts:207

Returns

boolean

---

classes

Get Signature

get classes(): LlamaClasses;

Defined in: bindings/Llama.ts:211

Returns

LlamaClasses

---

gpu

Get Signature

get gpu(): LlamaGpuType;

Defined in: bindings/Llama.ts:218

Returns

LlamaGpuType

---

supportsGpuOffloading

Get Signature

get supportsGpuOffloading(): boolean;

Defined in: bindings/Llama.ts:222

Returns

boolean

---

supportsMmap

Get Signature

get supportsMmap(): boolean;

Defined in: bindings/Llama.ts:226

Returns

boolean

---

gpuSupportsMmap

Get Signature

get gpuSupportsMmap(): boolean;

Defined in: bindings/Llama.ts:230

Returns

boolean

---

supportsMlock

Get Signature

get supportsMlock(): boolean;

Defined in: bindings/Llama.ts:234

Returns

boolean

---

cpuMathCores

Get Signature

get cpuMathCores(): number;

Defined in: bindings/Llama.ts:239

The number of CPU cores that are useful for math

Returns

number

---

maxThreads

Get Signature

get maxThreads(): number;

Defined in: bindings/Llama.ts:250

The maximum number of threads that can be used by the Llama instance.

If set to 0, the Llama instance will have no limit on the number of threads.

See the maxThreads option of getLlama for more information.

Returns

number

Set Signature

set maxThreads(value: number): void;

Defined in: bindings/Llama.ts:254

Parameters

Parameter	Type
value	number

Returns

void

---

numa

Get Signature

get numa(): LlamaNuma;

Defined in: bindings/Llama.ts:261

See the numa option of getLlama for more information

Returns

LlamaNuma

---

logLevel

Get Signature

get logLevel(): LlamaLogLevel;

Defined in: bindings/Llama.ts:265

Returns

LlamaLogLevel

Set Signature

set logLevel(value: LlamaLogLevel): void;

Defined in: bindings/Llama.ts:269

Parameters

Parameter	Type
value	LlamaLogLevel

Returns

void

---

logger

Get Signature

get logger(): (level: LlamaLogLevel, message: string) => void;

Defined in: bindings/Llama.ts:279

Returns

(level: LlamaLogLevel, message: string): void;

Parameters

Parameter	Type
level	LlamaLogLevel
message	string

Returns

void

Set Signature

set logger(value: (level: LlamaLogLevel, message: string) => void): void;

Defined in: bindings/Llama.ts:283

Parameters

Parameter	Type
value	(level: LlamaLogLevel, message: string) => void

Returns

void

---

buildType

Get Signature

get buildType(): "localBuild" | "prebuilt";

Defined in: bindings/Llama.ts:290

Returns

"localBuild" | "prebuilt"

---

cmakeOptions

Get Signature

get cmakeOptions(): Readonly<Record<string, string>>;

Defined in: bindings/Llama.ts:294

Returns

Readonly<Record<string, string>>

---

llamaCppRelease

Get Signature

get llamaCppRelease(): {
  repo: string;
  release: string;
};

Defined in: bindings/Llama.ts:298

Returns

{
  repo: string;
  release: string;
}

repo

readonly repo: string;

release

readonly release: string;

---

systemInfo

Get Signature

get systemInfo(): string;

Defined in: bindings/Llama.ts:302

Returns

string

---

vramPaddingSize

Get Signature

get vramPaddingSize(): number;

Defined in: bindings/Llama.ts:314

VRAM padding used for memory size calculations, as these calculations are not always accurate. This is set by default to ensure stability, but can be configured when you call getLlama.

See vramPadding on getLlama for more information.

Returns

number

---

ramPaddingSize

Get Signature

get ramPaddingSize(): number;

Defined in: bindings/Llama.ts:324

RAM padding used for memory size calculations, as these calculations are not always accurate. This is set by default to ensure stability, but can be configured when you call getLlama.

See ramPadding on getLlama for more information.

Returns

number

Methods

dispose()

dispose(): Promise<void>;

Defined in: bindings/Llama.ts:188

Returns

Promise<void>

---

getVramState()

getVramState(): Promise<{
  total: number;
  used: number;
  free: number;
  unifiedSize: number;
}>;

Defined in: bindings/Llama.ts:334

The total amount of VRAM that is currently being used.

unifiedSize represents the amount of VRAM that is shared between the CPU and GPU. On SoC devices, this is usually the same as total.

Returns

Promise<{ total: number; used: number; free: number; unifiedSize: number; }>

---

getRamState()

getRamState(): Promise<RamState>;

Defined in: bindings/Llama.ts:348

Get the state of the system RAM

Returns

Promise<RamState>

---

getSwapState()

getSwapState(): Promise<{
  maxSize: number;
  allocated: number;
  used: number;
}>;

Defined in: bindings/Llama.ts:366

Get the state of the swap memory.

maxSize - The maximum size of the swap memory that the system can allocate. If the swap size is dynamic (like on macOS), this will be Infinity.

allocated - The total size allocated by the system for swap memory.

used - The amount of swap memory that is currently being used from the allocated size.

On Windows, this will return the info for the page file.

Returns

Promise<{ maxSize: number; allocated: number; used: number; }>

---

getLlamaMemoryUsage()

getLlamaMemoryUsage(): Promise<{
  gpuVram: number;
  cpuRam: number;
}>;

Defined in: bindings/Llama.ts:395

Get the total memory usage of this Llama instance

Returns

Promise<{ gpuVram: number; cpuRam: number; }>

---

setVramCap()

setVramCap(bytes: number | null): Promise<void>;

Defined in: bindings/Llama.ts:416

Cap the amount of VRAM that this Llama instance is allowed to use in bytes. This is useful for constraining the resource usage of models and contexts created with the Llama instance.

Capping to a value that's too low may cause model loads and context creations to either fail or not fully offload to VRAM, causing inference to be significantly slower.

Setting a cap will only affect future model loads and context creations.

Use with caution. Setting to null disables the cap.

Defaults to null.

Parameters

Parameter	Type
bytes	number | null

Returns

Promise<void>

---

getVramCap()

getVramCap(): number | null;

Defined in: bindings/Llama.ts:431

Get the current VRAM cap in bytes. See `setVramCap` for more information.

Defaults to null, which means no cap is set.

Returns

number | null

---

setRamCap()

setRamCap(bytes: number | null): Promise<void>;

Defined in: bindings/Llama.ts:455

Cap the amount of RAM that this Llama instance is allowed to use in bytes. This is useful for constraining the resource usage of models and contexts created with the Llama instance.

Capping to a value that's too low may cause model loads and context creations to fail. Capping to any value will exclude swap from the resource calculations, so extremely large models may not load at all even if you have enough swap available.

Setting a cap will only affect future model loads and context creations.

On unified memory systems, capping the RAM may also effectively cap the VRAM, as they are shared. On such systems, it's recommended to either cap the VRAM or the RAM (but not both), and if you need to cap both then make sure to set the RAM cap to a value greater than the VRAM cap.

Note: You can detect a unified memory system by checking whether getVramState().unifiedSize is greater than 0.

Use with caution. Setting to null disables the cap.

Defaults to null.

Parameters

Parameter	Type
bytes	number | null

Returns

Promise<void>

---

getRamCap()

getRamCap(): number | null;

Defined in: bindings/Llama.ts:472

Get the current RAM cap in bytes. See `setRamCap` for more information.

Defaults to null, which means no cap is set.

Returns

number | null

---

getGpuDeviceNames()

getGpuDeviceNames(): Promise<string[]>;

Defined in: bindings/Llama.ts:476

Returns

Promise<string[]>

---

loadModel()

loadModel(options: LlamaModelOptions): Promise<LlamaModel>;

Defined in: bindings/Llama.ts:484

Parameters

Parameter	Type
options	LlamaModelOptions

Returns

Promise<LlamaModel>

---

createGrammarForJsonSchema()

createGrammarForJsonSchema<T, Defs>(schema: Readonly<T> & GbnfJsonSchema<Defs>): Promise<LlamaJsonSchemaGrammar<T, Defs>>;

Defined in: bindings/Llama.ts:504

Type Parameters

Type Parameter	Default type
T extends | GbnfJsonBasicSchema | GbnfJsonConstSchema | GbnfJsonEnumSchema | GbnfJsonBasicStringSchema | GbnfJsonFormatStringSchema | GbnfJsonOneOfSchema<Defs> | GbnfJsonObjectSchema<string, Defs> | GbnfJsonArraySchema<Defs> | GbnfJsonRefSchema<Defs>	-
Defs extends GbnfJsonDefList<Defs>	Record<any, any>

Parameters

Parameter	Type
schema	Readonly<T> & GbnfJsonSchema<Defs>

Returns

Promise<LlamaJsonSchemaGrammar<T, Defs>>

See

• Using a JSON Schema Grammar tutorial
• Reducing Hallucinations When Using JSON Schema Grammar tutorial

---

getGrammarFor()

getGrammarFor(type: 
  | "json"
  | "json_arr"
  | "english"
  | "list"
  | "c"
  | "arithmetic"
  | "japanese"
  | "chess"): Promise<LlamaGrammar>;

Defined in: bindings/Llama.ts:512

Parameters

Parameter	Type
type	| "json" | "json_arr" | "english" | "list" | "c" | "arithmetic" | "japanese" | "chess"

Returns

Promise<LlamaGrammar>

---

createGrammar()

createGrammar(options: LlamaGrammarOptions): Promise<LlamaGrammar>;

Defined in: bindings/Llama.ts:519

Parameters

Parameter	Type
options	LlamaGrammarOptions

Returns

Promise<LlamaGrammar>

See

Using Grammar tutorial

---

defaultConsoleLogger()

static defaultConsoleLogger(level: LlamaLogLevel, message: string): void;

Defined in: bindings/Llama.ts:786

Parameters

Parameter	Type
level	LlamaLogLevel
message	string

Returns

void