- Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/ (( is a N'2015-03-18 09:48:54.360' test")
- .addTerminator(" ", DROP)
- .addTerminator("(", PRESERVE)
- .addTerminator("\"", "\"", PRESERVE)
- .addTerminator("N'", "'", PRESERVE)
- .addTerminator("/*", "*/", DROP);
-
- assertTokenEquals("\"", "hello", tokenizer);
- assertTokenEquals("(", null, tokenizer);
- assertTokenEquals("(", null, tokenizer);
- assertTokenEquals("is", null, tokenizer);
- assertTokenEquals("a", null, tokenizer);
- assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
- assertTokenEquals("test", null, tokenizer);
+ Tokenizer tokenizer = new Tokenizer("\"hello world\" /** comment **/ (( is a N'2015-03-18 09:48:54.360' test");
+ tokenizer.addTerminator(DROP,"\\s");
+ tokenizer.addTerminator(PRESERVE,"\\(");
+ tokenizer.addTerminator(PRESERVE, "\\\".*\\\"");
+ tokenizer.addTerminator(PRESERVE, "N'.*'");
+ tokenizer.addTerminator(DROP,"/\\*.*\\*/");
+
+ assertTokenEquals("\"hello world\"", tokenizer);
+ assertTokenEquals("(", tokenizer);
+ assertTokenEquals("(", tokenizer);
+ assertTokenEquals("is", tokenizer);
+ assertTokenEquals("a", tokenizer);
+ assertTokenEquals("N'2015-03-18 09:48:54.360'", tokenizer);
+ assertTokenEquals("test", tokenizer);