Class: SpecialToken

Defined in: utils/LlamaText.ts:518

Constructors

new SpecialToken()

new SpecialToken(value: BuiltinSpecialTokenValue): SpecialToken

Defined in: utils/LlamaText.ts:521

Parameters

Parameter	Type
value	BuiltinSpecialTokenValue

Returns

SpecialToken

Properties

value

readonly value: BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:519

Methods

toString()

toString(): BuiltinSpecialTokenValue

Defined in: utils/LlamaText.ts:525

Returns

BuiltinSpecialTokenValue

---

tokenize()

tokenize(tokenizer: Tokenizer): Token[]

Defined in: utils/LlamaText.ts:529

Parameters

Parameter	Type
tokenizer	Tokenizer

Returns

Token[]

---

toJSON()

toJSON(): LlamaTextSpecialTokenJSON

Defined in: utils/LlamaText.ts:533

Returns

LlamaTextSpecialTokenJSON

---

fromJSON()

static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken

Defined in: utils/LlamaText.ts:557

Parameters

Parameter	Type
json	LlamaTextSpecialTokenJSON

Returns

SpecialToken

---

isSpecialTokenJSON()

static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON

Defined in: utils/LlamaText.ts:564

Parameters

Parameter	Type
value	LlamaTextJSONValue

Returns

value is LlamaTextSpecialTokenJSON

---

getTokenToValueMap()

static getTokenToValueMap(tokenizer: Tokenizer): ReadonlyMap<undefined | Token, BuiltinSpecialTokenValue>

Defined in: utils/LlamaText.ts:568

Parameters

Parameter	Type
tokenizer	Tokenizer

Returns

ReadonlyMap<undefined | Token, BuiltinSpecialTokenValue>