X-Git-Url: http://www2.svjatoslav.eu/gitweb/?a=blobdiff_plain;f=src%2Ftest%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizerTest.java;h=519b6d1248a725958df88705560170a1cb12bce2;hb=9bb7c6bf73ebbcbc66f4abd0fabf2f698c42d42c;hp=e40b40167f4308da24b31ce4fb2a6d3c6d2ead4d;hpb=4d4a86dce1b84d9ef26d95168f2fea5477cedaa5;p=svjatoslav_commons.git

diff --git a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
index e40b401..519b6d1 100644
--- a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
+++ b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
@@ -2,23 +2,50 @@ package eu.svjatoslav.commons.string.tokenizer;
 
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
+import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
+import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
+import static org.junit.Assert.*;
 
 public class TokenizerTest {
 
-
     @Test
-    public void peekNextToken() throws Exception {
-        Tokenizer tokenizer = new Tokenizer("this is a test")
-                .addTerminator(" ", Terminator.TerminationStrategy.DROP);
+    public void testPeeking() throws Exception {
+        Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test");
+        tokenizer.addTerminator(DROP, "\\s");
+        tokenizer.addTerminator(PRESERVE, "N'.*'");
 
-        tokenizer.expectAndConsumeNextToken("this");
+        tokenizer.expectAndConsumeNextStringToken("this");
 
         assertEquals("is", tokenizer.peekNextToken().token);
 
         assertEquals("is", tokenizer.peekNextToken().token);
 
-        assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
+        assertTrue(tokenizer.peekIsOneOf("maybe", "is", "that"));
+    }
+
+    @Test
+    public void testTokenization() throws Exception {
+        Tokenizer tokenizer = new Tokenizer("\"hello world\"   /** comment **/   ((  is a N'2015-03-18 09:48:54.360' test");
+        tokenizer.addTerminator(DROP,"\\s");
+        tokenizer.addTerminator(PRESERVE,"\\(");
+        tokenizer.addTerminator(PRESERVE, "\\\".*\\\"");
+        tokenizer.addTerminator(PRESERVE, "N'.*'");
+        tokenizer.addTerminator(DROP,"/\\*.*\\*/");
+
+        assertTokenEquals("\"hello world\"", tokenizer);
+        assertTokenEquals("(", tokenizer);
+        assertTokenEquals("(", tokenizer);
+        assertTokenEquals("is", tokenizer);
+        assertTokenEquals("a", tokenizer);
+        assertTokenEquals("N'2015-03-18 09:48:54.360'", tokenizer);
+        assertTokenEquals("test", tokenizer);
+
+        assertNull(tokenizer.getNextToken());
+        assertFalse(tokenizer.hasMoreContent());
+    }
+
+    private void assertTokenEquals(String expectedValue, Tokenizer tokenizer){
+        assertEquals(expectedValue, tokenizer.getNextToken().token);
     }
 
 }
\ No newline at end of file