1 package eu.svjatoslav.commons.string.tokenizer;
5 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
6 import static org.junit.Assert.assertEquals;
8 public class TokenizerTest {
10 public void findTokenTerminator() throws Exception {
12 Tokenizer tokenizer = new Tokenizer("this /* comment */ a test")
13 .addTerminator("/*", "*/", PRESERVE);
21 public void you_can_peek() throws Exception {
22 Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
23 .addTerminator(" ", Terminator.TerminationStrategy.DROP)
24 .addTerminator("N'", "'", PRESERVE);
26 tokenizer.expectAndConsumeNextToken("this");
28 assertEquals("is", tokenizer.peekNextToken().token);
30 assertEquals("is", tokenizer.peekNextToken().token);
32 assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
36 public void complexTerminator() throws Exception {
37 Tokenizer tokenizer = new Tokenizer(" this((\"hello\" /* comment */ (( is a N'2015-03-18 09:48:54.360' test")
38 .addTerminator(" ", Terminator.TerminationStrategy.DROP)
39 .addTerminator("(", Terminator.TerminationStrategy.PRESERVE)
40 .addTerminator("\"", "\"" ,Terminator.TerminationStrategy.PRESERVE)
41 .addTerminator("/*", "*/" ,Terminator.TerminationStrategy.DROP)
44 System.out.println(tokenizer.getNextToken().token);
45 System.out.println(tokenizer.getNextToken().token);
46 System.out.println(tokenizer.getNextToken().token);
47 System.out.println(tokenizer.getNextToken().token);
48 System.out.println(tokenizer.getNextToken().token);
49 System.out.println(tokenizer.getNextToken().token);
50 System.out.println(tokenizer.getNextToken().token);
51 System.out.println(tokenizer.getNextToken().token);
52 System.out.println(tokenizer.getNextToken().token);
55 // tokenizer.expectAndConsumeNextToken("this");
57 // assertEquals("is", tokenizer.peekNextToken().token);
59 // assertEquals("is", tokenizer.peekNextToken().token);
61 // assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
66 public void testComplexTerminator() throws Exception {
67 Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360' /* thoe unto u */ test")
68 .addTerminator(" ", Terminator.TerminationStrategy.DROP)
69 .addTerminator("/*", "*/", PRESERVE);
71 // tokenizer.expectAndConsumeNextToken("this");
73 // assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
75 System.out.println("1st: " + tokenizer.getNextToken().token);
77 System.out.println("2nd: " + tokenizer.getNextToken().token);
79 System.out.println("2nd: " + tokenizer.getNextToken().token);
81 System.out.println("2nd: " + tokenizer.getNextToken().token);
83 System.out.println("2nd: " + tokenizer.getNextToken().token);