onigleam/tokenizer

Types

Result of tokenization

pub type TokenizeResult {
  TokenizeResult(
    tokens: List(token.Token),
    flags: options.FlagProperties,
  )
}

TokenizeResult(
  tokens: List(token.Token),
  flags: options.FlagProperties,
)

pub fn tokenize(
  pattern: String,
  options: options.ParseOptions,
) -> Result(TokenizeResult, errors.TokenizeError)

Tokenize an Oniguruma regex pattern