Class: SpecialTokensText
Defined in: utils/LlamaText.ts:433
Constructors
Constructor
ts
new SpecialTokensText(value: string): SpecialTokensText;Defined in: utils/LlamaText.ts:436
Parameters
| Parameter | Type |
|---|---|
value | string |
Returns
SpecialTokensText
Properties
value
ts
readonly value: string;Defined in: utils/LlamaText.ts:434
Methods
toString()
ts
toString(): string;Defined in: utils/LlamaText.ts:440
Returns
string
tokenize()
ts
tokenize(tokenizer: Tokenizer, trimLeadingSpace: boolean): Token[];Defined in: utils/LlamaText.ts:444
Parameters
| Parameter | Type | Default value |
|---|---|---|
tokenizer | Tokenizer | undefined |
trimLeadingSpace | boolean | false |
Returns
Token[]
tokenizeSpecialTokensOnly()
ts
tokenizeSpecialTokensOnly(tokenizer: Tokenizer): (string | Token)[];Defined in: utils/LlamaText.ts:448
Parameters
| Parameter | Type |
|---|---|
tokenizer | Tokenizer |
Returns
(string | Token)[]
toJSON()
ts
toJSON(): LlamaTextSpecialTokensTextJSON;Defined in: utils/LlamaText.ts:471
Returns
LlamaTextSpecialTokensTextJSON
fromJSON()
ts
static fromJSON(json: LlamaTextSpecialTokensTextJSON): SpecialTokensText;Defined in: utils/LlamaText.ts:495
Parameters
| Parameter | Type |
|---|---|
json | LlamaTextSpecialTokensTextJSON |
Returns
SpecialTokensText
isSpecialTokensTextJSON()
ts
static isSpecialTokensTextJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokensTextJSON;Defined in: utils/LlamaText.ts:502
Parameters
| Parameter | Type |
|---|---|
value | LlamaTextJSONValue |
Returns
value is LlamaTextSpecialTokensTextJSON
wrapIf()
ts
static wrapIf(shouldWrap: boolean, value: string): string | SpecialTokensText;Defined in: utils/LlamaText.ts:509
Wraps the value with a SpecialTokensText only if shouldWrap is true
Parameters
| Parameter | Type |
|---|---|
shouldWrap | boolean |
value | string |
Returns
string | SpecialTokensText