X-Git-Url: http://www2.svjatoslav.eu/gitweb/?p=svjatoslav_commons.git;a=blobdiff_plain;f=src%2Ftest%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizerTest.java;h=19fbd4d1538563b835b7c2ec85a2c52eea36a78a;hp=519b6d1248a725958df88705560170a1cb12bce2;hb=18e9d2f537f5bb900172558b6e7af9485392f32e;hpb=9bb7c6bf73ebbcbc66f4abd0fabf2f698c42d42c

diff --git a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
index 519b6d1..19fbd4d 100644
--- a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
+++ b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
@@ -44,6 +44,23 @@ public class TokenizerTest {
         assertFalse(tokenizer.hasMoreContent());
     }
 
+
+    @Test
+    public void testMultilineTokenization() throws Exception {
+        Tokenizer tokenizer = new Tokenizer(
+                "* heading\r\nnormal text\r\nnormal text 2");
+        tokenizer.addTerminator(PRESERVE,"\\*[ \\t]+.*\\r?\\n");
+        tokenizer.addTerminator(DROP,"\\r?\\n", "normalText");
+
+        assertTokenEquals("* heading\r\n", tokenizer);
+        assertTokenEquals("normal text", tokenizer);
+        assertTokenEquals("normal text 2", tokenizer);
+
+        assertNull(tokenizer.getNextToken());
+        assertFalse(tokenizer.hasMoreContent());
+    }
+
+
     private void assertTokenEquals(String expectedValue, Tokenizer tokenizer){
         assertEquals(expectedValue, tokenizer.getNextToken().token);
     }