- @Test
- public void testComplexTerminator() throws Exception {
- Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360' /* thoe unto u */ test")
- .addTerminator(" ", DROP)
- .addTerminator("/*", "*/", PRESERVE);
-
-// tokenizer.expectAndConsumeNextToken("this");
-
-// assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
-
- System.out.println("1st: " + tokenizer.getNextToken().token);
-
- System.out.println("2nd: " + tokenizer.getNextToken().token);
-
- System.out.println("2nd: " + tokenizer.getNextToken().token);
-
- System.out.println("2nd: " + tokenizer.getNextToken().token);
-
- System.out.println("2nd: " + tokenizer.getNextToken().token);