Handle end of input. Speed improvements. Deleted legacy code.

[svjatoslav_commons.git] / src / test / java / eu / svjatoslav / commons / string / tokenizer / TokenizerTest.java

diff --git a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
index e40b401..84571e8 100644 (file)
--- a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
+++ b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
@@ -2,15 +2,19 @@ package eu.svjatoslav.commons.string.tokenizer;
 
 import org.junit.Test;
 
+import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
+import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 
 public class TokenizerTest {
 
-
     @Test
-    public void peekNextToken() throws Exception {
-        Tokenizer tokenizer = new Tokenizer("this is a test")
-                .addTerminator(" ", Terminator.TerminationStrategy.DROP);
+    public void testPeeking() throws Exception {
+        Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
+                .addTerminator(" ", DROP)
+                .addTerminator("N'", "'", PRESERVE);
 
         tokenizer.expectAndConsumeNextToken("this");
 
@@ -21,4 +25,45 @@ public class TokenizerTest {
         assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
     }
 
+    @Test
+    public void testTokenization() throws Exception {
+        Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/   ((  is a N'2015-03-18 09:48:54.360' test")
+                .addTerminator(" ", DROP)
+                .addTerminator("(", PRESERVE)
+                .addTerminator("\"", "\"" , PRESERVE)
+                .addTerminator("N'", "'" , PRESERVE)
+                .addTerminator("/*", "*/" , DROP)
+                ;
+
+        assertTokenEquals("\"", "hello", tokenizer);
+        assertTokenEquals("(", null, tokenizer);
+        assertTokenEquals("(", null, tokenizer);
+        assertTokenEquals("is", null, tokenizer);
+        assertTokenEquals("a", null, tokenizer);
+        assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
+        assertTokenEquals("test", null, tokenizer);
+
+        assertNull(tokenizer.getNextToken());
+        assertFalse(tokenizer.hasMoreTokens());
+    }
+
+    private void assertTokenEquals(String token, String reminder, Tokenizer tokenizer) throws InvalidSyntaxException {
+        TokenizerMatch nextToken = tokenizer.getNextToken();
+
+        assertEquals(token, nextToken.token);
+
+        if (reminder == null)
+            assertNull(nextToken.reminder);
+        else
+            assertEquals(reminder, nextToken.reminder);
+    }
+
+    private void debugNextToken(Tokenizer tokenizer) throws InvalidSyntaxException {
+        TokenizerMatch nextToken = tokenizer.getNextToken();
+        if (nextToken == null)
+            System.out.println("null");
+        else
+            System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
+    }
+    
 }
\ No newline at end of file