@Test
public void testPeeking() throws Exception {
- Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
- .addTerminator(" ", DROP)
- .addTerminator("N'", "'", PRESERVE);
+ Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test");
+ tokenizer.addTerminator(DROP, "\\s");
+ tokenizer.addTerminator(PRESERVE, "N'.*'");
- tokenizer.expectAndConsumeNextToken("this");
+ tokenizer.expectAndConsumeNextStringToken("this");
assertEquals("is", tokenizer.peekNextToken().token);
assertEquals("is", tokenizer.peekNextToken().token);
- assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
+ assertTrue(tokenizer.peekIsOneOf("maybe", "is", "that"));
}
@Test
public void testTokenization() throws Exception {
- Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/ (( is a N'2015-03-18 09:48:54.360' test")
- .addTerminator(" ", DROP)
- .addTerminator("(", PRESERVE)
- .addTerminator("\"", "\"", PRESERVE)
- .addTerminator("N'", "'", PRESERVE)
- .addTerminator("/*", "*/", DROP);
-
- assertTokenEquals("\"", "hello", tokenizer);
- assertTokenEquals("(", null, tokenizer);
- assertTokenEquals("(", null, tokenizer);
- assertTokenEquals("is", null, tokenizer);
- assertTokenEquals("a", null, tokenizer);
- assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
- assertTokenEquals("test", null, tokenizer);
+ Tokenizer tokenizer = new Tokenizer("\"hello world\" /** comment **/ (( is a N'2015-03-18 09:48:54.360' test");
+ tokenizer.addTerminator(DROP,"\\s");
+ tokenizer.addTerminator(PRESERVE,"\\(");
+ tokenizer.addTerminator(PRESERVE, "\\\".*\\\"");
+ tokenizer.addTerminator(PRESERVE, "N'.*'");
+ tokenizer.addTerminator(DROP,"/\\*.*\\*/");
+
+ assertTokenEquals("\"hello world\"", tokenizer);
+ assertTokenEquals("(", tokenizer);
+ assertTokenEquals("(", tokenizer);
+ assertTokenEquals("is", tokenizer);
+ assertTokenEquals("a", tokenizer);
+ assertTokenEquals("N'2015-03-18 09:48:54.360'", tokenizer);
+ assertTokenEquals("test", tokenizer);
assertNull(tokenizer.getNextToken());
assertFalse(tokenizer.hasMoreContent());
}
- private void assertTokenEquals(String token, String reminder, Tokenizer tokenizer) throws InvalidSyntaxException {
- TokenizerMatch nextToken = tokenizer.getNextToken();
- assertEquals(token, nextToken.token);
+ @Test
+ public void testMultilineTokenization() throws Exception {
+ Tokenizer tokenizer = new Tokenizer(
+ "* heading\r\nnormal text\r\nnormal text 2");
+ tokenizer.addTerminator(PRESERVE,"\\*[ \\t]+.*\\r?\\n");
+ tokenizer.addTerminator(DROP,"\\r?\\n", "normalText");
+
+ assertTokenEquals("* heading\r\n", tokenizer);
+ assertTokenEquals("normal text", tokenizer);
+ assertTokenEquals("normal text 2", tokenizer);
- if (reminder == null)
- assertNull(nextToken.reminder);
- else
- assertEquals(reminder, nextToken.reminder);
+ assertNull(tokenizer.getNextToken());
+ assertFalse(tokenizer.hasMoreContent());
}
- private void debugNextToken(Tokenizer tokenizer) throws InvalidSyntaxException {
- TokenizerMatch nextToken = tokenizer.getNextToken();
- if (nextToken == null)
- System.out.println("null");
- else
- System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
+
+ private void assertTokenEquals(String expectedValue, Tokenizer tokenizer){
+ assertEquals(expectedValue, tokenizer.getNextToken().token);
}
}
\ No newline at end of file