X-Git-Url: http://www2.svjatoslav.eu/gitweb/?a=blobdiff_plain;f=src%2Fmain%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizer.java;h=01216d8411db0229a3d218ddca001f154b32f330;hb=4d4a86dce1b84d9ef26d95168f2fea5477cedaa5;hp=a89677b0df47366c37aa4053b4295dda1a21c5d7;hpb=a338e6716fcfe8a78692c4d803ec6a51d50b20a7;p=svjatoslav_commons.git

diff --git a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
index a89677b..01216d8 100755
--- a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
+++ b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
@@ -13,6 +13,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Stack;
 
+import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
+
 public class Tokenizer {
 
     final Stack<Integer> tokenIndexes = new Stack<>();
@@ -33,19 +35,18 @@ public class Tokenizer {
     }
 
     public Tokenizer addTerminator(final String startSequence,
-                                   final boolean ignoreTerminator) {
-        terminators.add(new Terminator(startSequence, ignoreTerminator));
+                                   final Terminator.TerminationStrategy terminationStrategy) {
+        terminators.add(new Terminator(startSequence, terminationStrategy));
         return this;
     }
 
     public Tokenizer addTerminator(final String startSequence,
-                                   final String endSequence, final boolean ignoreTerminator) {
-        terminators.add(new Terminator(startSequence, endSequence,
-                ignoreTerminator));
+                                   final String endSequence, final Terminator.TerminationStrategy terminationStrategy) {
+        terminators.add(new Terminator(startSequence, endSequence, terminationStrategy));
         return this;
     }
 
-    public void expectNextToken(final String value)
+    public void expectAndConsumeNextToken(final String value)
             throws InvalidSyntaxException {
         final TokenizerMatch match = getNextToken();
         if (!value.equals(match.token))
@@ -66,7 +67,7 @@ public class Tokenizer {
             for (final Terminator terminator : terminators)
                 if (sequenceMatches(terminator.startSequence))
 
-                    if (terminator.ignoreTerminator) {
+                    if (terminator.termination == DROP) {
                         currentIndex += terminator.startSequence.length();
 
                         if (terminator.endSequence != null)
@@ -95,7 +96,7 @@ public class Tokenizer {
 
     }
 
-    public boolean isNextToken(final String token) {
+    public boolean consumeIfNextToken(final String token) {
         if (token.equals(getNextToken().token))
             return true;
 
@@ -103,6 +104,31 @@ public class Tokenizer {
         return false;
     }
 
+    public TokenizerMatch peekNextToken(){
+        TokenizerMatch result = getNextToken();
+        unreadToken();
+        return result;
+    }
+
+    public boolean peekIsOneOf(String ... possibilities){
+        String nextToken = peekNextToken().token;
+
+        for (String possibility : possibilities)
+            if (possibility.equals(nextToken))
+                return true;
+
+        return false;
+    }
+
+    public void peekExpectNoneOf(String ... possibilities) throws InvalidSyntaxException {
+        TokenizerMatch nextToken = peekNextToken();
+
+        for (String possibility : possibilities)
+            if (possibility.equals(nextToken))
+                throw new InvalidSyntaxException("Not expected \"" + nextToken + "\" here.");
+    }
+
+
     public boolean sequenceMatches(final String sequence) {
         if ((currentIndex + sequence.length()) > source.length())
             return false;