Skip to content

Class: Llama

Defined in: bindings/Llama.ts:35

Properties

onDispose

ts
readonly onDispose: EventRelay<void>;

Defined in: bindings/Llama.ts:72

Accessors

disposed

Get Signature

ts
get disposed(): boolean;

Defined in: bindings/Llama.ts:178

Returns

boolean


classes

Get Signature

ts
get classes(): LlamaClasses;

Defined in: bindings/Llama.ts:182

Returns

LlamaClasses


gpu

Get Signature

ts
get gpu(): LlamaGpuType;

Defined in: bindings/Llama.ts:189

Returns

LlamaGpuType


supportsGpuOffloading

Get Signature

ts
get supportsGpuOffloading(): boolean;

Defined in: bindings/Llama.ts:193

Returns

boolean


supportsMmap

Get Signature

ts
get supportsMmap(): boolean;

Defined in: bindings/Llama.ts:197

Returns

boolean


gpuSupportsMmap

Get Signature

ts
get gpuSupportsMmap(): boolean;

Defined in: bindings/Llama.ts:201

Returns

boolean


supportsMlock

Get Signature

ts
get supportsMlock(): boolean;

Defined in: bindings/Llama.ts:205

Returns

boolean


cpuMathCores

Get Signature

ts
get cpuMathCores(): number;

Defined in: bindings/Llama.ts:210

The number of CPU cores that are useful for math

Returns

number


maxThreads

Get Signature

ts
get maxThreads(): number;

Defined in: bindings/Llama.ts:221

The maximum number of threads that can be used by the Llama instance.

If set to 0, the Llama instance will have no limit on the number of threads.

See the maxThreads option of getLlama for more information.

Returns

number

Set Signature

ts
set maxThreads(value: number): void;

Defined in: bindings/Llama.ts:225

Parameters
ParameterType
valuenumber
Returns

void


numa

Get Signature

ts
get numa(): LlamaNuma;

Defined in: bindings/Llama.ts:232

See the numa option of getLlama for more information

Returns

LlamaNuma


logLevel

Get Signature

ts
get logLevel(): LlamaLogLevel;

Defined in: bindings/Llama.ts:236

Returns

LlamaLogLevel

Set Signature

ts
set logLevel(value: LlamaLogLevel): void;

Defined in: bindings/Llama.ts:240

Parameters
ParameterType
valueLlamaLogLevel
Returns

void


logger

Get Signature

ts
get logger(): (level: LlamaLogLevel, message: string) => void;

Defined in: bindings/Llama.ts:250

Returns
ts
(level: LlamaLogLevel, message: string): void;
Parameters
ParameterType
levelLlamaLogLevel
messagestring
Returns

void

Set Signature

ts
set logger(value: (level: LlamaLogLevel, message: string) => void): void;

Defined in: bindings/Llama.ts:254

Parameters
ParameterType
value(level: LlamaLogLevel, message: string) => void
Returns

void


buildType

Get Signature

ts
get buildType(): "localBuild" | "prebuilt";

Defined in: bindings/Llama.ts:261

Returns

"localBuild" | "prebuilt"


cmakeOptions

Get Signature

ts
get cmakeOptions(): Readonly<Record<string, string>>;

Defined in: bindings/Llama.ts:265

Returns

Readonly<Record<string, string>>


llamaCppRelease

Get Signature

ts
get llamaCppRelease(): {
  repo: string;
  release: string;
};

Defined in: bindings/Llama.ts:269

Returns
ts
{
  repo: string;
  release: string;
}
repo
ts
readonly repo: string;
release
ts
readonly release: string;

systemInfo

Get Signature

ts
get systemInfo(): string;

Defined in: bindings/Llama.ts:273

Returns

string


vramPaddingSize

Get Signature

ts
get vramPaddingSize(): number;

Defined in: bindings/Llama.ts:285

VRAM padding used for memory size calculations, as these calculations are not always accurate. This is set by default to ensure stability, but can be configured when you call getLlama.

See vramPadding on getLlama for more information.

Returns

number

Methods

dispose()

ts
dispose(): Promise<void>;

Defined in: bindings/Llama.ts:163

Returns

Promise<void>


getVramState()

ts
getVramState(): Promise<{
  total: number;
  used: number;
  free: number;
  unifiedSize: number;
}>;

Defined in: bindings/Llama.ts:295

The total amount of VRAM that is currently being used.

unifiedSize represents the amount of VRAM that is shared between the CPU and GPU. On SoC devices, this is usually the same as total.

Returns

Promise<{ total: number; used: number; free: number; unifiedSize: number; }>


getSwapState()

ts
getSwapState(): Promise<{
  maxSize: number;
  allocated: number;
  used: number;
}>;

Defined in: bindings/Llama.ts:320

Get the state of the swap memory.

maxSize - The maximum size of the swap memory that the system can allocate. If the swap size is dynamic (like on macOS), this will be Infinity.

allocated - The total size allocated by the system for swap memory.

used - The amount of swap memory that is currently being used from the allocated size.

On Windows, this will return the info for the page file.

Returns

Promise<{ maxSize: number; allocated: number; used: number; }>


getGpuDeviceNames()

ts
getGpuDeviceNames(): Promise<string[]>;

Defined in: bindings/Llama.ts:346

Returns

Promise<string[]>


loadModel()

ts
loadModel(options: LlamaModelOptions): Promise<LlamaModel>;

Defined in: bindings/Llama.ts:354

Parameters

ParameterType
optionsLlamaModelOptions

Returns

Promise<LlamaModel>


createGrammarForJsonSchema()

ts
createGrammarForJsonSchema<T, Defs>(schema: Readonly<T> & GbnfJsonSchema<Defs>): Promise<LlamaJsonSchemaGrammar<T, Defs>>;

Defined in: bindings/Llama.ts:374

Type Parameters

Type ParameterDefault type
T extends | GbnfJsonBasicSchema | GbnfJsonConstSchema | GbnfJsonEnumSchema | GbnfJsonBasicStringSchema | GbnfJsonFormatStringSchema | GbnfJsonOneOfSchema<Defs> | GbnfJsonObjectSchema<string, Defs> | GbnfJsonArraySchema<Defs> | GbnfJsonRefSchema<Defs>-
Defs extends GbnfJsonDefList<Defs>Record<any, any>

Parameters

ParameterType
schemaReadonly<T> & GbnfJsonSchema<Defs>

Returns

Promise<LlamaJsonSchemaGrammar<T, Defs>>

See


getGrammarFor()

ts
getGrammarFor(type: 
  | "json"
  | "json_arr"
  | "english"
  | "list"
  | "c"
  | "arithmetic"
  | "japanese"
  | "chess"): Promise<LlamaGrammar>;

Defined in: bindings/Llama.ts:382

Parameters

ParameterType
type| "json" | "json_arr" | "english" | "list" | "c" | "arithmetic" | "japanese" | "chess"

Returns

Promise<LlamaGrammar>


createGrammar()

ts
createGrammar(options: LlamaGrammarOptions): Promise<LlamaGrammar>;

Defined in: bindings/Llama.ts:389

Parameters

ParameterType
optionsLlamaGrammarOptions

Returns

Promise<LlamaGrammar>

See

Using Grammar tutorial


defaultConsoleLogger()

ts
static defaultConsoleLogger(level: LlamaLogLevel, message: string): void;

Defined in: bindings/Llama.ts:600

Parameters

ParameterType
levelLlamaLogLevel
messagestring

Returns

void