Class: SpecialToken

Defined in: utils/LlamaText.ts:522

Constructors

Constructor

new SpecialToken(value: BuiltinSpecialTokenValue): SpecialToken;

Defined in: utils/LlamaText.ts:525

Parameters

Parameter	Type
value	BuiltinSpecialTokenValue

Returns

SpecialToken

Properties

value

readonly value: BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:523

Methods

toString()

toString(): BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:529

Returns

BuiltinSpecialTokenValue

---

tokenize()

tokenize(tokenizer: Tokenizer): Token[];

Defined in: utils/LlamaText.ts:533

Parameters

Parameter	Type
tokenizer	Tokenizer

Returns

Token[]

---

toJSON()

toJSON(): LlamaTextSpecialTokenJSON;

Defined in: utils/LlamaText.ts:537

Returns

LlamaTextSpecialTokenJSON

---

fromJSON()

static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;

Defined in: utils/LlamaText.ts:561

Parameters

Parameter	Type
json	LlamaTextSpecialTokenJSON

Returns

SpecialToken

---

isSpecialTokenJSON()

static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;

Defined in: utils/LlamaText.ts:568

Parameters

Parameter	Type
value	LlamaTextJSONValue

Returns

value is LlamaTextSpecialTokenJSON

---

getTokenToValueMap()

static getTokenToValueMap(tokenizer: Tokenizer): ReadonlyMap<Token | undefined, BuiltinSpecialTokenValue>;

Defined in: utils/LlamaText.ts:572

Parameters

Parameter	Type
tokenizer	Tokenizer

Returns

ReadonlyMap<Token | undefined, BuiltinSpecialTokenValue>