Class: SpecialTokensText

Constructors

new SpecialTokensText()

new SpecialTokensText(value: string): SpecialTokensText

Parameters

Parameter	Type
value	string

Returns

SpecialTokensText

Defined in

utils/LlamaText.ts:433

Properties

value

readonly value: string;

Defined in

utils/LlamaText.ts:431

Methods

toString()

toString(): string

Returns

string

Defined in

utils/LlamaText.ts:437

---

tokenize()

tokenize(tokenizer: Tokenizer, trimLeadingSpace: boolean): Token[]

Parameters

Parameter	Type	Default value
tokenizer	Tokenizer	undefined
trimLeadingSpace	boolean	false

Returns

Token[]

Defined in

utils/LlamaText.ts:441

---

tokenizeSpecialTokensOnly()

tokenizeSpecialTokensOnly(tokenizer: Tokenizer): (string | Token)[]

Parameters

Parameter	Type
tokenizer	Tokenizer

Returns

(string | Token)[]

Defined in

utils/LlamaText.ts:445

---

toJSON()

toJSON(): LlamaTextSpecialTokensTextJSON

Returns

LlamaTextSpecialTokensTextJSON

Defined in

utils/LlamaText.ts:469

---

fromJSON()

static fromJSON(json: LlamaTextSpecialTokensTextJSON): SpecialTokensText

Parameters

Parameter	Type
json	LlamaTextSpecialTokensTextJSON

Returns

SpecialTokensText

Defined in

utils/LlamaText.ts:493

---

isSpecialTokensTextJSON()

static isSpecialTokensTextJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokensTextJSON

Parameters

Parameter	Type
value	LlamaTextJSONValue

Returns

value is LlamaTextSpecialTokensTextJSON

Defined in

utils/LlamaText.ts:500

---

wrapIf()

static wrapIf(shouldWrap: boolean, value: string): string | SpecialTokensText

Wraps the value with a SpecialTokensText only if shouldWrap is true

Parameters

Parameter	Type
shouldWrap	boolean
value	string

Returns

string | SpecialTokensText

Defined in

utils/LlamaText.ts:507