Class: SpecialToken

Defined in: utils/LlamaText.ts:519

Constructors

new SpecialToken()

new SpecialToken(value: BuiltinSpecialTokenValue): SpecialToken

Defined in: utils/LlamaText.ts:522

Parameters

Parameter	Type
value	BuiltinSpecialTokenValue

Returns

SpecialToken

Properties

value

readonly value: BuiltinSpecialTokenValue;

Defined in: utils/LlamaText.ts:520

Methods

toString()

toString(): BuiltinSpecialTokenValue

Defined in: utils/LlamaText.ts:526

Returns

BuiltinSpecialTokenValue

---

tokenize()

tokenize(tokenizer: Tokenizer): Token[]

Defined in: utils/LlamaText.ts:530

Parameters

Parameter	Type
tokenizer	Tokenizer

Returns

Token[]

---

toJSON()

toJSON(): LlamaTextSpecialTokenJSON

Defined in: utils/LlamaText.ts:534

Returns

LlamaTextSpecialTokenJSON

---

fromJSON()

static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken

Defined in: utils/LlamaText.ts:558

Parameters

Parameter	Type
json	LlamaTextSpecialTokenJSON

Returns

SpecialToken

---

isSpecialTokenJSON()

static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON

Defined in: utils/LlamaText.ts:565

Parameters

Parameter	Type
value	LlamaTextJSONValue

Returns

value is LlamaTextSpecialTokenJSON

---

getTokenToValueMap()

static getTokenToValueMap(tokenizer: Tokenizer): ReadonlyMap<undefined | Token, BuiltinSpecialTokenValue>

Defined in: utils/LlamaText.ts:569

Parameters

Parameter	Type
tokenizer	Tokenizer

Returns

ReadonlyMap<undefined | Token, BuiltinSpecialTokenValue>