Possibility to peek next value.

[svjatoslav_commons.git] / src / main / java / eu / svjatoslav / commons / string / tokenizer / Tokenizer.java

diff --git a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
index 1905d9a..ba1590d 100755 (executable)
--- a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
+++ b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
@@ -1,6 +1,6 @@
 /*
  * Svjatoslav Commons - shared library of common functionality.
- * Copyright ©2012-2014, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu
+ * Copyright ©2012-2017, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of version 3 of the GNU Lesser General Public License
@@ -13,26 +13,37 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Stack;
 
+import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
+
 public class Tokenizer {
 
     final Stack<Integer> tokenIndexes = new Stack<>();
     private final List<Terminator> terminators = new ArrayList<>();
-    private final String source;
+    private String source;
     private int currentIndex = 0;
 
     public Tokenizer(final String source) {
         this.source = source;
     }
 
-    public void addTerminator(final String startSequence,
-                              final boolean ignoreTerminator) {
-        terminators.add(new Terminator(startSequence, ignoreTerminator));
+    public Tokenizer(){}
+
+    public Tokenizer setSource(String source){
+        this.source = source;
+        currentIndex = 0;
+        return this;
     }
 
-    public void addTerminator(final String startSequence,
-                              final String endSequence, final boolean ignoreTerminator) {
-        terminators.add(new Terminator(startSequence, endSequence,
-                ignoreTerminator));
+    public Tokenizer addTerminator(final String startSequence,
+                                   final Terminator.TerminationStrategy terminationStrategy) {
+        terminators.add(new Terminator(startSequence, terminationStrategy));
+        return this;
+    }
+
+    public Tokenizer addTerminator(final String startSequence,
+                                   final String endSequence, final Terminator.TerminationStrategy terminationStrategy) {
+        terminators.add(new Terminator(startSequence, endSequence, terminationStrategy));
+        return this;
     }
 
     public void expectNextToken(final String value)
@@ -56,7 +67,7 @@ public class Tokenizer {
             for (final Terminator terminator : terminators)
                 if (sequenceMatches(terminator.startSequence))
 
-                    if (terminator.ignoreTerminator) {
+                    if (terminator.termination == DROP) {
                         currentIndex += terminator.startSequence.length();
 
                         if (terminator.endSequence != null)
@@ -85,7 +96,7 @@ public class Tokenizer {
 
     }
 
-    public boolean probeNextToken(final String token) {
+    public boolean consumeIfNextToken(final String token) {
         if (token.equals(getNextToken().token))
             return true;
 
@@ -93,6 +104,23 @@ public class Tokenizer {
         return false;
     }
 
+    public TokenizerMatch peekNextToken(){
+        TokenizerMatch result = getNextToken();
+        unreadToken();
+        return result;
+    }
+
+    public boolean peekIsOneOf(String [] ... possibilities){
+        TokenizerMatch nextToken = peekNextToken();
+
+        for (String[] possibility : possibilities)
+            if (possibility.equals(nextToken))
+                return true;
+
+        return false;
+    }
+
+
     public boolean sequenceMatches(final String sequence) {
         if ((currentIndex + sequence.length()) > source.length())
             return false;