Skip to content

Class: SpecialToken

Defined in: utils/LlamaText.ts:518

Constructors

Constructor

ts
new SpecialToken(value: BuiltinSpecialTokenValue): SpecialToken;

Defined in: utils/LlamaText.ts:521

Parameters

ParameterType
valueBuiltinSpecialTokenValue

Returns

SpecialToken

Properties

value

ts
readonly value: BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:519

Methods

toString()

ts
toString(): BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:525

Returns

BuiltinSpecialTokenValue


tokenize()

ts
tokenize(tokenizer: Tokenizer): Token[];

Defined in: utils/LlamaText.ts:529

Parameters

ParameterType
tokenizerTokenizer

Returns

Token[]


toJSON()

ts
toJSON(): LlamaTextSpecialTokenJSON;

Defined in: utils/LlamaText.ts:533

Returns

LlamaTextSpecialTokenJSON


fromJSON()

ts
static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;

Defined in: utils/LlamaText.ts:557

Parameters

ParameterType
jsonLlamaTextSpecialTokenJSON

Returns

SpecialToken


isSpecialTokenJSON()

ts
static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;

Defined in: utils/LlamaText.ts:564

Parameters

ParameterType
valueLlamaTextJSONValue

Returns

value is LlamaTextSpecialTokenJSON


getTokenToValueMap()

ts
static getTokenToValueMap(tokenizer: Tokenizer): ReadonlyMap<undefined | Token, BuiltinSpecialTokenValue>;

Defined in: utils/LlamaText.ts:568

Parameters

ParameterType
tokenizerTokenizer

Returns

ReadonlyMap<undefined | Token, BuiltinSpecialTokenValue>