X-Git-Url: http://www2.svjatoslav.eu/gitweb/?p=svjatoslav_commons.git;a=blobdiff_plain;f=src%2Ftest%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizerTest.java;h=84571e898a5c5807b0f2c2f8a9ca54d5aa568c29;hp=ddb2662bfd8c81022313135f3889ad09eb42c820;hb=ba10494483d7eaf1e9d58ddb1402806c8fc58178;hpb=3bc3db3ceb288b82e48c349bf27dfafda2bcd444

diff --git a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
index ddb2662..84571e8 100644
--- a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
+++ b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
@@ -5,21 +5,13 @@ import org.junit.Test;
 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 
 public class TokenizerTest {
-    @Test
-    public void findTokenTerminator() throws Exception {
-
-        Tokenizer tokenizer = new Tokenizer("this /* comment */ a test")
-                .addTerminator("/*", "*/", PRESERVE);
-
-
-
-    }
-
 
     @Test
-    public void you_can_peek() throws Exception {
+    public void testPeeking() throws Exception {
         Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
                 .addTerminator(" ", DROP)
                 .addTerminator("N'", "'", PRESERVE);
@@ -34,56 +26,44 @@ public class TokenizerTest {
     }
 
     @Test
-    public void complexTerminator() throws Exception {
-        Tokenizer tokenizer = new Tokenizer("/* hello */ /** comment **/   ((  is a N'2015-03-18 09:48:54.360' test")
+    public void testTokenization() throws Exception {
+        Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/   ((  is a N'2015-03-18 09:48:54.360' test")
                 .addTerminator(" ", DROP)
                 .addTerminator("(", PRESERVE)
                 .addTerminator("\"", "\"" , PRESERVE)
-                .addTerminator("/*", "*/" , PRESERVE)
+                .addTerminator("N'", "'" , PRESERVE)
+                .addTerminator("/*", "*/" , DROP)
                 ;
 
-        TokenizerMatch nextToken = tokenizer.getNextToken();
-        System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
-        System.out.println(tokenizer.getNextToken().token);
-        System.out.println(tokenizer.getNextToken().token);
-        System.out.println(tokenizer.getNextToken().token);
-        System.out.println(tokenizer.getNextToken().token);
-        System.out.println(tokenizer.getNextToken().token);
-        System.out.println(tokenizer.getNextToken().token);
-        System.out.println(tokenizer.getNextToken().token);
-        System.out.println(tokenizer.getNextToken().token);
-
+        assertTokenEquals("\"", "hello", tokenizer);
+        assertTokenEquals("(", null, tokenizer);
+        assertTokenEquals("(", null, tokenizer);
+        assertTokenEquals("is", null, tokenizer);
+        assertTokenEquals("a", null, tokenizer);
+        assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
+        assertTokenEquals("test", null, tokenizer);
 
-//        tokenizer.expectAndConsumeNextToken("this");
-//
-//        assertEquals("is", tokenizer.peekNextToken().token);
-//
-//        assertEquals("is", tokenizer.peekNextToken().token);
-//
-//        assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
+        assertNull(tokenizer.getNextToken());
+        assertFalse(tokenizer.hasMoreTokens());
     }
 
+    private void assertTokenEquals(String token, String reminder, Tokenizer tokenizer) throws InvalidSyntaxException {
+        TokenizerMatch nextToken = tokenizer.getNextToken();
 
-    @Test
-    public void testComplexTerminator() throws Exception {
-        Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360'  /* thoe unto u */ test")
-                .addTerminator(" ", DROP)
-                .addTerminator("/*", "*/", PRESERVE);
-
-//        tokenizer.expectAndConsumeNextToken("this");
-
-//        assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
-
-        System.out.println("1st: " + tokenizer.getNextToken().token);
-
-        System.out.println("2nd: " + tokenizer.getNextToken().token);
-
-        System.out.println("2nd: " + tokenizer.getNextToken().token);
-
-        System.out.println("2nd: " + tokenizer.getNextToken().token);
-
-        System.out.println("2nd: " + tokenizer.getNextToken().token);
+        assertEquals(token, nextToken.token);
 
+        if (reminder == null)
+            assertNull(nextToken.reminder);
+        else
+            assertEquals(reminder, nextToken.reminder);
     }
 
+    private void debugNextToken(Tokenizer tokenizer) throws InvalidSyntaxException {
+        TokenizerMatch nextToken = tokenizer.getNextToken();
+        if (nextToken == null)
+            System.out.println("null");
+        else
+            System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
+    }
+    
 }
\ No newline at end of file