X-Git-Url: http://www2.svjatoslav.eu/gitweb/?a=blobdiff_plain;f=src%2Ftest%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizerTest.java;h=19fbd4d1538563b835b7c2ec85a2c52eea36a78a;hb=18e9d2f537f5bb900172558b6e7af9485392f32e;hp=ae6838691da35f789c9a635edf9524eb0202337f;hpb=c378ab1acf1d7299a572e4a5fc38996ee0408602;p=svjatoslav_commons.git

diff --git a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
index ae68386..19fbd4d 100644
--- a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
+++ b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
@@ -10,57 +10,59 @@ public class TokenizerTest {
 
     @Test
     public void testPeeking() throws Exception {
-        Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
-                .addTerminator(" ", DROP)
-                .addTerminator("N'", "'", PRESERVE);
+        Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test");
+        tokenizer.addTerminator(DROP, "\\s");
+        tokenizer.addTerminator(PRESERVE, "N'.*'");
 
-        tokenizer.expectAndConsumeNextToken("this");
+        tokenizer.expectAndConsumeNextStringToken("this");
 
         assertEquals("is", tokenizer.peekNextToken().token);
 
         assertEquals("is", tokenizer.peekNextToken().token);
 
-        assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
+        assertTrue(tokenizer.peekIsOneOf("maybe", "is", "that"));
     }
 
     @Test
     public void testTokenization() throws Exception {
-        Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/   ((  is a N'2015-03-18 09:48:54.360' test")
-                .addTerminator(" ", DROP)
-                .addTerminator("(", PRESERVE)
-                .addTerminator("\"", "\"", PRESERVE)
-                .addTerminator("N'", "'", PRESERVE)
-                .addTerminator("/*", "*/", DROP);
-
-        assertTokenEquals("\"", "hello", tokenizer);
-        assertTokenEquals("(", null, tokenizer);
-        assertTokenEquals("(", null, tokenizer);
-        assertTokenEquals("is", null, tokenizer);
-        assertTokenEquals("a", null, tokenizer);
-        assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
-        assertTokenEquals("test", null, tokenizer);
+        Tokenizer tokenizer = new Tokenizer("\"hello world\"   /** comment **/   ((  is a N'2015-03-18 09:48:54.360' test");
+        tokenizer.addTerminator(DROP,"\\s");
+        tokenizer.addTerminator(PRESERVE,"\\(");
+        tokenizer.addTerminator(PRESERVE, "\\\".*\\\"");
+        tokenizer.addTerminator(PRESERVE, "N'.*'");
+        tokenizer.addTerminator(DROP,"/\\*.*\\*/");
+
+        assertTokenEquals("\"hello world\"", tokenizer);
+        assertTokenEquals("(", tokenizer);
+        assertTokenEquals("(", tokenizer);
+        assertTokenEquals("is", tokenizer);
+        assertTokenEquals("a", tokenizer);
+        assertTokenEquals("N'2015-03-18 09:48:54.360'", tokenizer);
+        assertTokenEquals("test", tokenizer);
 
         assertNull(tokenizer.getNextToken());
-        assertFalse(tokenizer.hasMoreTokens());
+        assertFalse(tokenizer.hasMoreContent());
     }
 
-    private void assertTokenEquals(String token, String reminder, Tokenizer tokenizer) throws InvalidSyntaxException {
-        TokenizerMatch nextToken = tokenizer.getNextToken();
 
-        assertEquals(token, nextToken.token);
+    @Test
+    public void testMultilineTokenization() throws Exception {
+        Tokenizer tokenizer = new Tokenizer(
+                "* heading\r\nnormal text\r\nnormal text 2");
+        tokenizer.addTerminator(PRESERVE,"\\*[ \\t]+.*\\r?\\n");
+        tokenizer.addTerminator(DROP,"\\r?\\n", "normalText");
+
+        assertTokenEquals("* heading\r\n", tokenizer);
+        assertTokenEquals("normal text", tokenizer);
+        assertTokenEquals("normal text 2", tokenizer);
 
-        if (reminder == null)
-            assertNull(nextToken.reminder);
-        else
-            assertEquals(reminder, nextToken.reminder);
+        assertNull(tokenizer.getNextToken());
+        assertFalse(tokenizer.hasMoreContent());
     }
 
-    private void debugNextToken(Tokenizer tokenizer) throws InvalidSyntaxException {
-        TokenizerMatch nextToken = tokenizer.getNextToken();
-        if (nextToken == null)
-            System.out.println("null");
-        else
-            System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
+
+    private void assertTokenEquals(String expectedValue, Tokenizer tokenizer){
+        assertEquals(expectedValue, tokenizer.getNextToken().token);
     }
 
 }
\ No newline at end of file