src/main/java/eu/svjatoslav/inspector/tokenizer/Tokenizer.java

   1 package eu.svjatoslav.inspector.tokenizer;
   2
   3 import java.util.ArrayList;
   4 import java.util.List;
   5
   6 public class Tokenizer {
   7
   8         private final List<Terminator> terminators = new ArrayList<Terminator>();
   9         private final String source;
  10
  11         private int currentIndex = 0;
  12
  13         public Tokenizer(final String source) {
  14                 this.source = source;
  15         }
  16
  17         public void addTerminator(final String terminator, final boolean empty) {
  18
  19                 terminators.add(new Terminator(terminator, empty));
  20         }
  21
  22         public TokenizerMatch getToken() {
  23                 final StringBuffer result = new StringBuffer();
  24
  25                 while (true) {
  26                         if (currentIndex >= source.length())
  27                                 return null;
  28
  29                         boolean accumulateCurrentChar = true;
  30
  31                         findTerminator: for (final Terminator terminator : terminators)
  32                                 if (terminatorMatches(terminator))
  33                                         // empty space detected
  34                                         if (terminator.empty) {
  35                                                 currentIndex += terminator.value.length();
  36                                                 if (result.length() > 0)
  37                                                         return new TokenizerMatch(result.toString(),
  38                                                                         terminator);
  39                                                 else {
  40                                                         accumulateCurrentChar = false;
  41                                                         break findTerminator;
  42                                                 }
  43                                         } else if (result.length() > 0)
  44                                                 return new TokenizerMatch(result.toString(), terminator);
  45                                         else {
  46                                                 currentIndex += terminator.value.length();
  47                                                 return new TokenizerMatch(terminator.value, terminator);
  48                                         }
  49
  50                         if (accumulateCurrentChar) {
  51                                 result.append(source.charAt(currentIndex));
  52                                 currentIndex++;
  53                         }
  54                 }
  55
  56         }
  57
  58         public boolean terminatorMatches(final Terminator terminator) {
  59                 if ((currentIndex + terminator.value.length()) > source.length())
  60                         return false;
  61
  62                 for (int i = 0; i < terminator.value.length(); i++)
  63                         if (terminator.value.charAt(i) != source.charAt(i + currentIndex))
  64                                 return false;
  65
  66                 return true;
  67         }
  68
  69 }