node.js bindings for llama.cpp, and much more
Structured Decisions with any model
Experience the ease of running models on your machine
npx -y node-llama-cpp chatTo chat with models using a UI, try the example Electron app
Check out your hardware capabilities
npx -y node-llama-cpp inspect gpuEverything you need to use large language models in your project
Integrate node-llama-cpp in your codebase and prompt models
import {fileURLToPath} from "url";
import path from "path";
import {getLlama, LlamaChatSession} from "node-llama-cpp";
const __dirname = path.dirname(
fileURLToPath(import.meta.url)
);
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: path.join(__dirname, "my-model.gguf")
});
const context = await model.createContext();
const session = new LlamaChatSession({
contextSequence: context.getSequence()
});
const q1 = "Hi there, how are you?";
console.log("User: " + q1);
const a1 = await session.prompt(q1);
console.log("AI: " + a1);Get embedding for a given text
import {fileURLToPath} from "url";
import path from "path";
import {getLlama} from "node-llama-cpp";
const __dirname = path.dirname(
fileURLToPath(import.meta.url)
);
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: path.join(__dirname, "my-model.gguf")
});
const context = await model.createEmbeddingContext();
const text = "Hello world";
console.log("Text:", text);
const embedding = await context.getEmbeddingFor(text);
console.log("Embedding vector:", embedding.vector);Force a model response to follow your JSON schema
import {fileURLToPath} from "url";
import path from "path";
import {getLlama, LlamaChatSession} from "node-llama-cpp";
const __dirname = path.dirname(
fileURLToPath(import.meta.url)
);
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: path.join(__dirname, "my-model.gguf")
});
const context = await model.createContext();
const session = new LlamaChatSession({
contextSequence: context.getSequence()
});
const grammar = await llama.createGrammarForJsonSchema({
type: "object",
properties: {
positiveWordsInUserMessage: {
type: "array",
items: {
type: "string"
}
},
userMessagePositivityScoreFromOneToTen: {
enum: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
},
nameOfUser: {
oneOf: [{
type: "null"
}, {
type: "string"
}]
}
}
});
const prompt = "Hi there! I'm John. Nice to meet you!";
const res = await session.prompt(prompt, {
grammar
});
const parsedRes = grammar.parse(res);
console.log("User name:", parsedRes.nameOfUser);
console.log(
"Positive words in user message:",
parsedRes.positiveWordsInUserMessage
);
console.log(
"User message positivity score:",
parsedRes.userMessagePositivityScoreFromOneToTen
);Let a model call functions to retrieve data or perform actions
import {fileURLToPath} from "url";
import path from "path";
import {
getLlama,
LlamaChatSession,
defineChatSessionFunction
} from "node-llama-cpp";
const __dirname = path.dirname(
fileURLToPath(import.meta.url)
);
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: path.join(__dirname, "my-model.gguf")
});
const context = await model.createContext();
const session = new LlamaChatSession({
contextSequence: context.getSequence()
});
const fruitPrices: Record<string, string> = {
"apple": "$6",
"banana": "$4"
};
const functions = {
getFruitPrice: defineChatSessionFunction({
description: "Get the price of a fruit",
params: {
type: "object",
properties: {
name: {
type: "string"
}
}
},
async handler(params) {
const name = params.name.toLowerCase();
if (Object.keys(fruitPrices).includes(name))
return {
name: name,
price: fruitPrices[name]
};
return `Unrecognized fruit "${params.name}"`;
}
})
};
const q1 = "Is an apple more expensive than a banana?";
console.log("User: " + q1);
const a1 = await session.prompt(q1, {functions});
console.log("AI: " + a1);Classify text using a predefined set of options blazingly fast
import {getLlama, resolveModelFile} from "node-llama-cpp";
const modelUri = "hf:giladgd/gemma-4-E2B-it-GGUF:Q8_0";
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: await resolveModelFile(modelUri)
});
const context = await model.createDecisionContext();
await context.warmup(); // optional
const startTime = Date.now();
const ticket =
"I can't sign in after resetting my password. " +
"My whole team is locked out.";
const answers = await context.decide(ticket, {
troubleshooted: {
type: "noul",
instruction:
"Has the customer tried fixing the issue?"
},
team: {
type: "choice",
instruction:
"Which support team should handle this?",
criteria: {
accounts: "Signing in, account access",
billing: "Invoices, payments, refunds",
technical: "Problems after signing in"
}
},
impact: {
type: "score",
instruction:
"How much is the issue " +
"affecting the customer's work?",
criteria: [
"No interruption to work",
"Some tasks are slower or harder",
"The customer cannot continue their work"
]
}
});
console.log({
queue: answers.team.confidence >= 0.7
? answers.team.choice
: "triage",
priority: answers.impact.score >= 1.5
? "high"
: "normal",
nextStep: answers.troubleshooted.value >= 0.8
? "review previous attempts"
: "suggest initial troubleshooting"
});
const duration = Date.now() - startTime;
console.log("Duration: " + duration + "ms"); // 170msClassify an entire chat in place almost instantly
import {
getLlama,
resolveModelFile,
LlamaChatSession
} from "node-llama-cpp";
const modelUri = "hf:giladgd/gemma-4-E2B-it-GGUF:Q8_0";
const llama = await getLlama();
const model = await llama.loadModel({
modelPath: await resolveModelFile(modelUri)
});
const context = await model.createContext();
const session = new LlamaChatSession({
contextSequence: context.getSequence(),
systemPrompt:
"You help customers resolve account access issues"
});
await session.prompt(
"I can't sign in after resetting my password.",
{maxTokens: 150}
);
await session.prompt(
"That fixed it, thanks!",
{maxTokens: 100}
);
const startTime = Date.now();
const answers = await session.decide({
resolved: {
type: "noul",
instruction:
"Has the customer confirmed " +
"that their issue is resolved?"
},
needsHuman: {
type: "noul",
instruction:
"Has the customer asked to speak to a person?"
}
});
if (answers.needsHuman.value >= 0.8)
console.log("Hand off to a support agent");
else if (answers.resolved.value >= 0.8)
console.log("Mark the ticket as resolved");
else
console.log("Continue the conversation");
const duration = Date.now() - startTime;
console.log("Duration: " + duration + "ms"); // 123ms