import org.junit.Test;
+import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
import static org.junit.Assert.assertEquals;
@Test
public void you_can_peek() throws Exception {
Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
- .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+ .addTerminator(" ", DROP)
.addTerminator("N'", "'", PRESERVE);
tokenizer.expectAndConsumeNextToken("this");
@Test
public void complexTerminator() throws Exception {
- Tokenizer tokenizer = new Tokenizer(" this((\"hello\" /* comment */ (( is a N'2015-03-18 09:48:54.360' test")
- .addTerminator(" ", Terminator.TerminationStrategy.DROP)
- .addTerminator("(", Terminator.TerminationStrategy.PRESERVE)
- .addTerminator("\"", "\"" ,Terminator.TerminationStrategy.PRESERVE)
- .addTerminator("/*", "*/" ,Terminator.TerminationStrategy.DROP)
+ Tokenizer tokenizer = new Tokenizer("/* hello */ /** comment **/ (( is a N'2015-03-18 09:48:54.360' test")
+ .addTerminator(" ", DROP)
+ .addTerminator("(", PRESERVE)
+ .addTerminator("\"", "\"" , PRESERVE)
+ .addTerminator("/*", "*/" , PRESERVE)
;
- System.out.println(tokenizer.getNextToken().token);
+ TokenizerMatch nextToken = tokenizer.getNextToken();
+ System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
System.out.println(tokenizer.getNextToken().token);
System.out.println(tokenizer.getNextToken().token);
System.out.println(tokenizer.getNextToken().token);
@Test
public void testComplexTerminator() throws Exception {
Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360' /* thoe unto u */ test")
- .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+ .addTerminator(" ", DROP)
.addTerminator("/*", "*/", PRESERVE);
// tokenizer.expectAndConsumeNextToken("this");