Class: SpecialTokensText

Defined in: utils/LlamaText.ts:433

Constructors

new SpecialTokensText()

new SpecialTokensText(value: string): SpecialTokensText

Defined in: utils/LlamaText.ts:436

Parameters

Parameter	Type
value	string

Returns

SpecialTokensText

Properties

value

readonly value: string;

Defined in: utils/LlamaText.ts:434

Methods

toString()

toString(): string

Defined in: utils/LlamaText.ts:440

Returns

string

---

tokenize()

tokenize(tokenizer: Tokenizer, trimLeadingSpace: boolean): Token[]

Defined in: utils/LlamaText.ts:444

Parameters

Parameter	Type	Default value
tokenizer	Tokenizer	undefined
trimLeadingSpace	boolean	false

Returns

Token[]

---

tokenizeSpecialTokensOnly()

tokenizeSpecialTokensOnly(tokenizer: Tokenizer): (string | Token)[]

Defined in: utils/LlamaText.ts:448

Parameters

Parameter	Type
tokenizer	Tokenizer

Returns

(string | Token)[]

---

toJSON()

toJSON(): LlamaTextSpecialTokensTextJSON

Defined in: utils/LlamaText.ts:472

Returns

LlamaTextSpecialTokensTextJSON

---

fromJSON()

static fromJSON(json: LlamaTextSpecialTokensTextJSON): SpecialTokensText

Defined in: utils/LlamaText.ts:496

Parameters

Parameter	Type
json	LlamaTextSpecialTokensTextJSON

Returns

SpecialTokensText

---

isSpecialTokensTextJSON()

static isSpecialTokensTextJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokensTextJSON

Defined in: utils/LlamaText.ts:503

Parameters

Parameter	Type
value	LlamaTextJSONValue

Returns

value is LlamaTextSpecialTokensTextJSON

---

wrapIf()

static wrapIf(shouldWrap: boolean, value: string): string | SpecialTokensText

Defined in: utils/LlamaText.ts:510

Wraps the value with a SpecialTokensText only if shouldWrap is true

Parameters

Parameter	Type
shouldWrap	boolean
value	string

Returns

string | SpecialTokensText