1 package eu.svjatoslav.commons.string.tokenizer;
5 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
6 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
7 import static org.junit.Assert.assertEquals;
8 import static org.junit.Assert.assertFalse;
9 import static org.junit.Assert.assertNull;
11 public class TokenizerTest {
14 public void testPeeking() throws Exception {
15 Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
16 .addTerminator(" ", DROP)
17 .addTerminator("N'", "'", PRESERVE);
19 tokenizer.expectAndConsumeNextToken("this");
21 assertEquals("is", tokenizer.peekNextToken().token);
23 assertEquals("is", tokenizer.peekNextToken().token);
25 assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
29 public void testTokenization() throws Exception {
30 Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/ (( is a N'2015-03-18 09:48:54.360' test")
31 .addTerminator(" ", DROP)
32 .addTerminator("(", PRESERVE)
33 .addTerminator("\"", "\"" , PRESERVE)
34 .addTerminator("N'", "'" , PRESERVE)
35 .addTerminator("/*", "*/" , DROP)
38 assertTokenEquals("\"", "hello", tokenizer);
39 assertTokenEquals("(", null, tokenizer);
40 assertTokenEquals("(", null, tokenizer);
41 assertTokenEquals("is", null, tokenizer);
42 assertTokenEquals("a", null, tokenizer);
43 assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
44 assertTokenEquals("test", null, tokenizer);
46 assertNull(tokenizer.getNextToken());
47 assertFalse(tokenizer.hasMoreTokens());
50 private void assertTokenEquals(String token, String reminder, Tokenizer tokenizer) throws InvalidSyntaxException {
51 TokenizerMatch nextToken = tokenizer.getNextToken();
53 assertEquals(token, nextToken.token);
56 assertNull(nextToken.reminder);
58 assertEquals(reminder, nextToken.reminder);
61 private void debugNextToken(Tokenizer tokenizer) throws InvalidSyntaxException {
62 TokenizerMatch nextToken = tokenizer.getNextToken();
63 if (nextToken == null)
64 System.out.println("null");
66 System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");