1 package eu.svjatoslav.commons.string.tokenizer;
5 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
6 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
7 import static org.junit.Assert.*;
9 public class TokenizerTest {
12 public void testPeeking() throws Exception {
13 Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test");
14 tokenizer.addTerminator(DROP, "\\s");
15 tokenizer.addTerminator(PRESERVE, "N'.*'");
17 tokenizer.expectAndConsumeNextStringToken("this");
19 assertEquals("is", tokenizer.peekNextToken().token);
21 assertEquals("is", tokenizer.peekNextToken().token);
23 assertTrue(tokenizer.peekIsOneOf("maybe", "is", "that"));
27 public void testTokenization() throws Exception {
28 Tokenizer tokenizer = new Tokenizer("\"hello world\" /** comment **/ (( is a N'2015-03-18 09:48:54.360' test");
29 tokenizer.addTerminator(DROP,"\\s");
30 tokenizer.addTerminator(PRESERVE,"\\(");
31 tokenizer.addTerminator(PRESERVE, "\\\".*\\\"");
32 tokenizer.addTerminator(PRESERVE, "N'.*'");
33 tokenizer.addTerminator(DROP,"/\\*.*\\*/");
35 assertTokenEquals("\"hello world\"", tokenizer);
36 assertTokenEquals("(", tokenizer);
37 assertTokenEquals("(", tokenizer);
38 assertTokenEquals("is", tokenizer);
39 assertTokenEquals("a", tokenizer);
40 assertTokenEquals("N'2015-03-18 09:48:54.360'", tokenizer);
41 assertTokenEquals("test", tokenizer);
43 assertNull(tokenizer.getNextToken());
44 assertFalse(tokenizer.hasMoreContent());
47 private void assertTokenEquals(String expectedValue, Tokenizer tokenizer){
48 assertEquals(expectedValue, tokenizer.getNextToken().token);