1 package eu.svjatoslav.commons.string.tokenizer;
5 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
6 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
7 import static org.junit.Assert.assertEquals;
9 public class TokenizerTest {
11 public void findTokenTerminator() throws Exception {
13 Tokenizer tokenizer = new Tokenizer("this /* comment */ a test")
14 .addTerminator("/*", "*/", PRESERVE);
22 public void you_can_peek() throws Exception {
23 Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
24 .addTerminator(" ", DROP)
25 .addTerminator("N'", "'", PRESERVE);
27 tokenizer.expectAndConsumeNextToken("this");
29 assertEquals("is", tokenizer.peekNextToken().token);
31 assertEquals("is", tokenizer.peekNextToken().token);
33 assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
37 public void complexTerminator() throws Exception {
38 Tokenizer tokenizer = new Tokenizer("/* hello */ /** comment **/ (( is a N'2015-03-18 09:48:54.360' test")
39 .addTerminator(" ", DROP)
40 .addTerminator("(", PRESERVE)
41 .addTerminator("\"", "\"" , PRESERVE)
42 .addTerminator("/*", "*/" , PRESERVE)
45 TokenizerMatch nextToken = tokenizer.getNextToken();
46 System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
47 System.out.println(tokenizer.getNextToken().token);
48 System.out.println(tokenizer.getNextToken().token);
49 System.out.println(tokenizer.getNextToken().token);
50 System.out.println(tokenizer.getNextToken().token);
51 System.out.println(tokenizer.getNextToken().token);
52 System.out.println(tokenizer.getNextToken().token);
53 System.out.println(tokenizer.getNextToken().token);
54 System.out.println(tokenizer.getNextToken().token);
57 // tokenizer.expectAndConsumeNextToken("this");
59 // assertEquals("is", tokenizer.peekNextToken().token);
61 // assertEquals("is", tokenizer.peekNextToken().token);
63 // assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
68 public void testComplexTerminator() throws Exception {
69 Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360' /* thoe unto u */ test")
70 .addTerminator(" ", DROP)
71 .addTerminator("/*", "*/", PRESERVE);
73 // tokenizer.expectAndConsumeNextToken("this");
75 // assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
77 System.out.println("1st: " + tokenizer.getNextToken().token);
79 System.out.println("2nd: " + tokenizer.getNextToken().token);
81 System.out.println("2nd: " + tokenizer.getNextToken().token);
83 System.out.println("2nd: " + tokenizer.getNextToken().token);
85 System.out.println("2nd: " + tokenizer.getNextToken().token);