Class: SpecialToken
Defined in: utils/LlamaText.ts:518
Constructors
Constructor
ts
new SpecialToken(value: BuiltinSpecialTokenValue): SpecialToken;
Defined in: utils/LlamaText.ts:521
Parameters
Parameter | Type |
---|---|
value | BuiltinSpecialTokenValue |
Returns
SpecialToken
Properties
value
ts
readonly value: BuiltinSpecialTokenValue;
Defined in: utils/LlamaText.ts:519
Methods
toString()
ts
toString(): BuiltinSpecialTokenValue;
Defined in: utils/LlamaText.ts:525
Returns
tokenize()
ts
tokenize(tokenizer: Tokenizer): Token[];
Defined in: utils/LlamaText.ts:529
Parameters
Parameter | Type |
---|---|
tokenizer | Tokenizer |
Returns
Token
[]
toJSON()
ts
toJSON(): LlamaTextSpecialTokenJSON;
Defined in: utils/LlamaText.ts:533
Returns
fromJSON()
ts
static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;
Defined in: utils/LlamaText.ts:557
Parameters
Parameter | Type |
---|---|
json | LlamaTextSpecialTokenJSON |
Returns
SpecialToken
isSpecialTokenJSON()
ts
static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;
Defined in: utils/LlamaText.ts:564
Parameters
Parameter | Type |
---|---|
value | LlamaTextJSONValue |
Returns
value is LlamaTextSpecialTokenJSON
getTokenToValueMap()
ts
static getTokenToValueMap(tokenizer: Tokenizer): ReadonlyMap<undefined | Token, BuiltinSpecialTokenValue>;
Defined in: utils/LlamaText.ts:568
Parameters
Parameter | Type |
---|---|
tokenizer | Tokenizer |
Returns
ReadonlyMap
<undefined
| Token
, BuiltinSpecialTokenValue
>