node-llama-cpp

Appearance

Class: SpecialToken ​

Defined in: utils/LlamaText.ts:522

Constructors ​

Constructor ​

ts
new SpecialToken(value: BuiltinSpecialTokenValue): SpecialToken;

Defined in: utils/LlamaText.ts:525

Parameters ​

ParameterType
valueBuiltinSpecialTokenValue

Returns ​

SpecialToken

Properties ​

value ​

ts
readonly value: BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:523

Methods ​

toString() ​

ts
toString(): BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:529

Returns ​

BuiltinSpecialTokenValue

tokenize() ​

ts
tokenize(tokenizer: Tokenizer): Token[];

Defined in: utils/LlamaText.ts:533

Parameters ​

ParameterType
tokenizerTokenizer

Returns ​

Token[]

toJSON() ​

ts
toJSON(): LlamaTextSpecialTokenJSON;

Defined in: utils/LlamaText.ts:537

Returns ​

LlamaTextSpecialTokenJSON

fromJSON() ​

ts
static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;

Defined in: utils/LlamaText.ts:561

Parameters ​

ParameterType
jsonLlamaTextSpecialTokenJSON

Returns ​

SpecialToken

isSpecialTokenJSON() ​

ts
static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;

Defined in: utils/LlamaText.ts:568

Parameters ​

ParameterType
valueLlamaTextJSONValue

Returns ​

value is LlamaTextSpecialTokenJSON

getTokenToValueMap() ​

ts
static getTokenToValueMap(tokenizer: Tokenizer): ReadonlyMap<Token | undefined, BuiltinSpecialTokenValue>;

Defined in: utils/LlamaText.ts:572

Parameters ​

ParameterType
tokenizerTokenizer

Returns ​

ReadonlyMap<Token | undefined, BuiltinSpecialTokenValue>