- StringBuilder token = new StringBuilder();
-
- while (true){
- if (isTokenTermination()){
- Terminator tokenTerminator = findTokenTerminator();
-
- if (tokenTerminator.termination == PRESERVE){
- if (hasAccumulatedToken(token)){
- // already assembled some token
- return new TokenizerMatch(token.toString(), "", tokenTerminator);
- } else {
- currentIndex++;
- return new TokenizerMatch(tokenTerminator.startSequence, "", tokenTerminator);
- }
- } else if (tokenTerminator.termination == DROP){
- if (hasAccumulatedToken(token)){
- currentIndex++;
- return new TokenizerMatch(token.toString(), "", tokenTerminator);
- } else {
- currentIndex++;
- }
- }
- } else {
- token.append(source.charAt(currentIndex));
+ StringBuilder tokenAccumulator = new StringBuilder();
+
+ while (true) {
+
+ if (currentIndex >= source.length()) { // reached end of input
+ if (hasAccumulatedToken(tokenAccumulator))
+ return new TokenizerMatch(tokenAccumulator.toString(), null, null, this);
+ else
+ return null;
+ }
+
+ TokenizerMatch matchResult = findTerminatorMatch();
+ if (matchResult == null) {
+ tokenAccumulator.append(source.charAt(currentIndex));