Type Alias: Tokenizer
ts
type Tokenizer = {
tokenize: Token[];
}["tokenize"] & {
detokenize: Detokenizer;
isSpecialToken: boolean;
isEogToken: boolean;
};Defined in: types.ts:12
Type declaration
detokenize
ts
readonly detokenize: Detokenizer;isSpecialToken()
ts
isSpecialToken(token: Token): boolean;Parameters
| Parameter | Type |
|---|---|
token | Token |
Returns
boolean
isEogToken()
ts
isEogToken(token: Token): boolean;Parameters
| Parameter | Type |
|---|---|
token | Token |
Returns
boolean