- .addTerminator("\"", "\"" , PRESERVE)
- .addTerminator("/*", "*/" , PRESERVE)
- ;
-
- TokenizerMatch nextToken = tokenizer.getNextToken();
- System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
-
-
-// tokenizer.expectAndConsumeNextToken("this");
-//
-// assertEquals("is", tokenizer.peekNextToken().token);
-//
-// assertEquals("is", tokenizer.peekNextToken().token);
-//
-// assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
+ .addTerminator("\"", "\"", PRESERVE)
+ .addTerminator("N'", "'", PRESERVE)
+ .addTerminator("/*", "*/", DROP);
+
+ assertTokenEquals("\"", "hello", tokenizer);
+ assertTokenEquals("(", null, tokenizer);
+ assertTokenEquals("(", null, tokenizer);
+ assertTokenEquals("is", null, tokenizer);
+ assertTokenEquals("a", null, tokenizer);
+ assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
+ assertTokenEquals("test", null, tokenizer);
+
+ assertNull(tokenizer.getNextToken());
+ assertFalse(tokenizer.hasMoreContent());