assertFalse(tokenizer.hasMoreContent());
}
+
+ @Test
+ public void testMultilineTokenization() throws Exception {
+ Tokenizer tokenizer = new Tokenizer(
+ "* heading\r\nnormal text\r\nnormal text 2");
+ tokenizer.addTerminator(PRESERVE,"\\*[ \\t]+.*\\r?\\n");
+ tokenizer.addTerminator(DROP,"\\r?\\n", "normalText");
+
+ assertTokenEquals("* heading\r\n", tokenizer);
+ assertTokenEquals("normal text", tokenizer);
+ assertTokenEquals("normal text 2", tokenizer);
+
+ assertNull(tokenizer.getNextToken());
+ assertFalse(tokenizer.hasMoreContent());
+ }
+
+
private void assertTokenEquals(String expectedValue, Tokenizer tokenizer){
assertEquals(expectedValue, tokenizer.getNextToken().token);
}