Type Alias: Tokenizer

type Tokenizer = {
  tokenize: Token[];
}["tokenize"] & {
  detokenize: Detokenizer;
  isSpecialToken: boolean;
  isEogToken: boolean;
};

Defined in: types.ts:13

Type Declaration

detokenize

readonly detokenize: Detokenizer;

isSpecialToken()

isSpecialToken(token: Token): boolean;

Parameters

Parameter	Type
token	Token

Returns

boolean

isEogToken()

isEogToken(token: Token): boolean;

Parameters

Parameter	Type
token	Token

Returns

boolean