Added possibility to skip token stream until the end

diff --git a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
index 80f9c4f..c80aeb1 100755 (executable)
--- a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
+++ b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
@@ -198,4 +198,9 @@ public class Tokenizer {
         currentIndex = tokenIndexes.pop();
     }
 
+    public void skipUntilDataEnd() {
+        tokenIndexes.push(currentIndex);
+        currentIndex = source.length();
+    }
+
 }