+ @Test
+ public void complexTerminator() throws Exception {
+ Tokenizer tokenizer = new Tokenizer(" this((\"hello\" /* comment */ (( is a N'2015-03-18 09:48:54.360' test")
+ .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+ .addTerminator("(", Terminator.TerminationStrategy.PRESERVE)
+ .addTerminator("\"", "\"" ,Terminator.TerminationStrategy.PRESERVE)
+ .addTerminator("/*", "*/" ,Terminator.TerminationStrategy.DROP)
+ ;
+
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+
+
+// tokenizer.expectAndConsumeNextToken("this");
+//
+// assertEquals("is", tokenizer.peekNextToken().token);
+//
+// assertEquals("is", tokenizer.peekNextToken().token);
+//
+// assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
+ }
+
+
+ @Test
+ public void testComplexTerminator() throws Exception {
+ Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360' /* thoe unto u */ test")
+ .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+ .addTerminator("/*", "*/", PRESERVE);
+
+// tokenizer.expectAndConsumeNextToken("this");
+
+// assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
+
+ System.out.println("1st: " + tokenizer.getNextToken().token);
+
+ System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+ System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+ System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+ System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+ }
+