node-llama-cpp

Appearance

Class: SpecialToken ​

Defined in: utils/LlamaText.ts:518

Constructors ​

Constructor ​

ts
new SpecialToken(value: BuiltinSpecialTokenValue): SpecialToken;

Defined in: utils/LlamaText.ts:521

Parameters ​

ParameterType
valueBuiltinSpecialTokenValue

Returns ​

SpecialToken

Properties ​

value ​

ts
readonly value: BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:519

Methods ​

toString() ​

ts
toString(): BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:525

Returns ​

BuiltinSpecialTokenValue

tokenize() ​

ts
tokenize(tokenizer: Tokenizer): Token[];

Defined in: utils/LlamaText.ts:529

Parameters ​

ParameterType
tokenizerTokenizer

Returns ​

Token[]

toJSON() ​

ts
toJSON(): LlamaTextSpecialTokenJSON;

Defined in: utils/LlamaText.ts:533

Returns ​

LlamaTextSpecialTokenJSON

fromJSON() ​

ts
static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;

Defined in: utils/LlamaText.ts:557

Parameters ​

ParameterType
jsonLlamaTextSpecialTokenJSON

Returns ​

SpecialToken

isSpecialTokenJSON() ​

ts
static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;

Defined in: utils/LlamaText.ts:564

Parameters ​

ParameterType
valueLlamaTextJSONValue

Returns ​

value is LlamaTextSpecialTokenJSON

getTokenToValueMap() ​

ts
static getTokenToValueMap(tokenizer: Tokenizer): ReadonlyMap<undefined | Token, BuiltinSpecialTokenValue>;

Defined in: utils/LlamaText.ts:568

Parameters ​

ParameterType
tokenizerTokenizer

Returns ​

ReadonlyMap<undefined | Token, BuiltinSpecialTokenValue>