1 package eu.svjatoslav.inspector.tokenizer;
3 import java.util.ArrayList;
5 import java.util.Stack;
7 public class Tokenizer {
9 private final List<Terminator> terminators = new ArrayList<Terminator>();
10 private final String source;
12 Stack<Integer> tokenIndexes = new Stack<Integer>();
14 private int currentIndex = 0;
16 public Tokenizer(final String source) {
20 public void addTerminator(final String startSequence,
21 final boolean ignoreTerminator) {
22 terminators.add(new Terminator(startSequence, ignoreTerminator));
25 public void addTerminator(final String startSequence,
26 final String endSequence, final boolean ignoreTerminator) {
27 terminators.add(new Terminator(startSequence, endSequence,
31 public void expectNextToken(final String value)
32 throws InvalidSyntaxException {
33 final TokenizerMatch match = getNextToken();
34 if (!value.equals(match.token))
35 throw new InvalidSyntaxException("Expected \"" + value
36 + "\" but got \"" + match.token + "\" instead.");
39 public TokenizerMatch getNextToken() {
40 tokenIndexes.push(currentIndex);
41 final StringBuffer result = new StringBuffer();
44 if (currentIndex >= source.length())
47 boolean accumulateCurrentChar = true;
49 findTerminator: for (final Terminator terminator : terminators)
50 if (sequenceMatches(terminator.startSequence))
52 if (terminator.ignoreTerminator) {
53 currentIndex += terminator.startSequence.length();
55 if (terminator.endSequence != null)
56 skipUntilSequence(terminator.endSequence);
58 if (result.length() > 0)
59 return new TokenizerMatch(result.toString(),
62 accumulateCurrentChar = false;
65 } else if (result.length() > 0)
66 return new TokenizerMatch(result.toString(), terminator);
68 currentIndex += terminator.startSequence.length();
69 return new TokenizerMatch(terminator.startSequence,
73 if (accumulateCurrentChar) {
74 result.append(source.charAt(currentIndex));
81 public boolean probeNextToken(final String token) {
82 if (token.equals(getNextToken().token))
89 public boolean sequenceMatches(final String sequence) {
90 if ((currentIndex + sequence.length()) > source.length())
93 for (int i = 0; i < sequence.length(); i++)
94 if (sequence.charAt(i) != source.charAt(i + currentIndex))
100 public void skipUntilDataEnd() {
101 tokenIndexes.push(currentIndex);
102 currentIndex = source.length();
105 public void skipUntilSequence(final String sequence) {
106 while (currentIndex < source.length()) {
107 if (sequenceMatches(sequence)) {
108 currentIndex += sequence.length();
116 public void unreadToken() {
117 currentIndex = tokenIndexes.pop();