Type Alias: Tokenizer

type Tokenizer = {
  tokenize: Token[];
 }["tokenize"] & {
  detokenize: Detokenizer;
  isSpecialToken: boolean;
  isEogToken: boolean;
};

Defined in: types.ts:12

Type declaration

detokenize

readonly detokenize: Detokenizer;

isSpecialToken()

Parameters

Parameter	Type
token	Token

Returns

boolean

isEogToken()

Parameters

Parameter	Type
token	Token

Returns

boolean