Type Alias: Tokenizer

type Tokenizer: {
  tokenize: Token[];
 }["tokenize"] & {
  detokenize: Detokenizer;
  isSpecialToken: boolean;
  isEogToken: boolean;
};

Type declaration

detokenize

readonly detokenize: Detokenizer;

isSpecialToken()

Parameters

Parameter	Type
token	Token

Returns

boolean

isEogToken()

Parameters

Parameter	Type
token	Token

Returns

boolean

Defined in

types.ts:12