- final StringBuilder result = new StringBuilder();
-
- while (true) {
- if (currentIndex >= source.length())
- return null;
-
- boolean accumulateCurrentChar = true;
-
- for (final Terminator terminator : terminators)
- if (sequenceMatches(terminator.startSequence))
-
- if (terminator.ignoreTerminator) {
- currentIndex += terminator.startSequence.length();
-
- if (terminator.endSequence != null)
- skipUntilSequence(terminator.endSequence);
-
- if (result.length() > 0)
- return new TokenizerMatch(result.toString(),
- terminator);
- else {
- accumulateCurrentChar = false;
- break;
- }
- } else if (result.length() > 0)
- return new TokenizerMatch(result.toString(), terminator);
- else {
- currentIndex += terminator.startSequence.length();
- return new TokenizerMatch(terminator.startSequence,
- terminator);
- }
-
- if (accumulateCurrentChar) {
- result.append(source.charAt(currentIndex));
+
+ StringBuilder tokenAccumulator = new StringBuilder();
+
+ while (true){
+
+ if (currentIndex >= source.length()){ // reached end of input
+ if (hasAccumulatedToken(tokenAccumulator))
+ return new TokenizerMatch(tokenAccumulator.toString(), null, null);
+ else
+ return null;
+ }
+
+ if (isOngoingToken()) {
+ tokenAccumulator.append(source.charAt(currentIndex));