Convenience method for splitting string into groups based on regular expression

[svjatoslav_commons.git] / src / test / java / eu / svjatoslav / commons / string / tokenizer / TokenizerTest.java

diff --git a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
index 519b6d1..19fbd4d 100644 (file)
--- a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
+++ b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java
@@ -44,6 +44,23 @@ public class TokenizerTest {
         assertFalse(tokenizer.hasMoreContent());
     }
 
+
+    @Test
+    public void testMultilineTokenization() throws Exception {
+        Tokenizer tokenizer = new Tokenizer(
+                "* heading\r\nnormal text\r\nnormal text 2");
+        tokenizer.addTerminator(PRESERVE,"\\*[ \\t]+.*\\r?\\n");
+        tokenizer.addTerminator(DROP,"\\r?\\n", "normalText");
+
+        assertTokenEquals("* heading\r\n", tokenizer);
+        assertTokenEquals("normal text", tokenizer);
+        assertTokenEquals("normal text 2", tokenizer);
+
+        assertNull(tokenizer.getNextToken());
+        assertFalse(tokenizer.hasMoreContent());
+    }
+
+
     private void assertTokenEquals(String expectedValue, Tokenizer tokenizer){
         assertEquals(expectedValue, tokenizer.getNextToken().token);
     }