public class Tokenizer
extends java.lang.Object
| Modifier and Type | Method and Description |
|---|---|
static java.util.List<Token> |
tokenize(java.lang.String text,
java.lang.String fileName) |
static java.util.List<Token> |
tokenize(java.lang.String text,
java.lang.String fileName,
boolean skipWhitespaces,
boolean errorOnUnbalancedStringQuotes,
boolean errorOnIncompleteEscapeChars) |
public static java.util.List<Token> tokenize(java.lang.String text, java.lang.String fileName)
public static java.util.List<Token> tokenize(java.lang.String text, java.lang.String fileName, boolean skipWhitespaces, boolean errorOnUnbalancedStringQuotes, boolean errorOnIncompleteEscapeChars)