Class: SpecialTokensText

Defined in: utils/LlamaText.ts:437

Constructors

Constructor

new SpecialTokensText(value: string): SpecialTokensText;

Defined in: utils/LlamaText.ts:440

Parameters

Parameter	Type
value	string

Returns

SpecialTokensText

Properties

value

readonly value: string;

Defined in: utils/LlamaText.ts:438

Methods

toString()

toString(): string;

Defined in: utils/LlamaText.ts:444

Returns

string

---

tokenize()

tokenize(tokenizer: Tokenizer, trimLeadingSpace?: boolean): Token[];

Defined in: utils/LlamaText.ts:448

Parameters

Parameter	Type	Default value
tokenizer	Tokenizer	undefined
trimLeadingSpace	boolean	false

Returns

Token[]

---

tokenizeSpecialTokensOnly()

tokenizeSpecialTokensOnly(tokenizer: Tokenizer): (string | Token)[];

Defined in: utils/LlamaText.ts:452

Parameters

Parameter	Type
tokenizer	Tokenizer

Returns

(string | Token)[]

---

toJSON()

toJSON(): LlamaTextSpecialTokensTextJSON;

Defined in: utils/LlamaText.ts:475

Returns

LlamaTextSpecialTokensTextJSON

---

fromJSON()

static fromJSON(json: LlamaTextSpecialTokensTextJSON): SpecialTokensText;

Defined in: utils/LlamaText.ts:499

Parameters

Parameter	Type
json	LlamaTextSpecialTokensTextJSON

Returns

SpecialTokensText

---

isSpecialTokensTextJSON()

static isSpecialTokensTextJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokensTextJSON;

Defined in: utils/LlamaText.ts:506

Parameters

Parameter	Type
value	LlamaTextJSONValue

Returns

value is LlamaTextSpecialTokensTextJSON

---

wrapIf()

static wrapIf(shouldWrap: boolean, value: string): string | SpecialTokensText;

Defined in: utils/LlamaText.ts:513

Wraps the value with a SpecialTokensText only if shouldWrap is true

Parameters

Parameter	Type
shouldWrap	boolean
value	string

Returns

string | SpecialTokensText