X-Git-Url: http://www2.svjatoslav.eu/gitweb/?p=svjatoslav_commons.git;a=blobdiff_plain;f=src%2Ftest%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizerTest.java;h=e72b9363bb22c327ef3cac0b3aaf19e25395be22;hp=e40b40167f4308da24b31ce4fb2a6d3c6d2ead4d;hb=798a165906826270924f0d5d58cfa04cf6bdc113;hpb=2082c4e93c58ae61eac0bdf7955e8996f6a19768

diff --git a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
index e40b401..e72b936 100644
--- a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
+++ b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
@@ -2,15 +2,26 @@ package eu.svjatoslav.commons.string.tokenizer;
 
 import org.junit.Test;
 
+import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
 import static org.junit.Assert.assertEquals;
 
 public class TokenizerTest {
+    @Test
+    public void findTokenTerminator() throws Exception {
+
+        Tokenizer tokenizer = new Tokenizer("this /* comment */ a test")
+                .addTerminator("/*", "*/", PRESERVE);
+
+
+
+    }
 
 
     @Test
-    public void peekNextToken() throws Exception {
-        Tokenizer tokenizer = new Tokenizer("this is a test")
-                .addTerminator(" ", Terminator.TerminationStrategy.DROP);
+    public void you_can_peek() throws Exception {
+        Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
+                .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+                .addTerminator("N'", "'", PRESERVE);
 
         tokenizer.expectAndConsumeNextToken("this");
 
@@ -21,4 +32,56 @@ public class TokenizerTest {
         assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
     }
 
+    @Test
+    public void complexTerminator() throws Exception {
+        Tokenizer tokenizer = new Tokenizer("   this((\"hello\"  /* comment */   ((  is a N'2015-03-18 09:48:54.360' test")
+                .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+                .addTerminator("(", Terminator.TerminationStrategy.PRESERVE)
+                .addTerminator("\"", "\"" ,Terminator.TerminationStrategy.PRESERVE)
+                .addTerminator("/*", "*/" ,Terminator.TerminationStrategy.DROP)
+                ;
+
+        System.out.println(tokenizer.getNextToken().token);
+        System.out.println(tokenizer.getNextToken().token);
+        System.out.println(tokenizer.getNextToken().token);
+        System.out.println(tokenizer.getNextToken().token);
+        System.out.println(tokenizer.getNextToken().token);
+        System.out.println(tokenizer.getNextToken().token);
+        System.out.println(tokenizer.getNextToken().token);
+        System.out.println(tokenizer.getNextToken().token);
+        System.out.println(tokenizer.getNextToken().token);
+
+
+//        tokenizer.expectAndConsumeNextToken("this");
+//
+//        assertEquals("is", tokenizer.peekNextToken().token);
+//
+//        assertEquals("is", tokenizer.peekNextToken().token);
+//
+//        assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
+    }
+
+
+    @Test
+    public void testComplexTerminator() throws Exception {
+        Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360'  /* thoe unto u */ test")
+                .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+                .addTerminator("/*", "*/", PRESERVE);
+
+//        tokenizer.expectAndConsumeNextToken("this");
+
+//        assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
+
+        System.out.println("1st: " + tokenizer.getNextToken().token);
+
+        System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+        System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+        System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+        System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+    }
+
 }
\ No newline at end of file