import org.junit.Test;
+import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
import static org.junit.Assert.assertEquals;
public class TokenizerTest {
+ @Test
+ public void findTokenTerminator() throws Exception {
+
+ Tokenizer tokenizer = new Tokenizer("this /* comment */ a test")
+ .addTerminator("/*", "*/", PRESERVE);
+
+
+
+ }
@Test
- public void peekNextToken() throws Exception {
- Tokenizer tokenizer = new Tokenizer("this is a test")
- .addTerminator(" ", Terminator.TerminationStrategy.DROP);
+ public void you_can_peek() throws Exception {
+ Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
+ .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+ .addTerminator("N'", "'", PRESERVE);
tokenizer.expectAndConsumeNextToken("this");
assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
}
+ @Test
+ public void complexTerminator() throws Exception {
+ Tokenizer tokenizer = new Tokenizer(" this((\"hello\" /* comment */ (( is a N'2015-03-18 09:48:54.360' test")
+ .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+ .addTerminator("(", Terminator.TerminationStrategy.PRESERVE)
+ .addTerminator("\"", "\"" ,Terminator.TerminationStrategy.PRESERVE)
+ .addTerminator("/*", "*/" ,Terminator.TerminationStrategy.DROP)
+ ;
+
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+ System.out.println(tokenizer.getNextToken().token);
+
+
+// tokenizer.expectAndConsumeNextToken("this");
+//
+// assertEquals("is", tokenizer.peekNextToken().token);
+//
+// assertEquals("is", tokenizer.peekNextToken().token);
+//
+// assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
+ }
+
+
+ @Test
+ public void testComplexTerminator() throws Exception {
+ Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360' /* thoe unto u */ test")
+ .addTerminator(" ", Terminator.TerminationStrategy.DROP)
+ .addTerminator("/*", "*/", PRESERVE);
+
+// tokenizer.expectAndConsumeNextToken("this");
+
+// assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
+
+ System.out.println("1st: " + tokenizer.getNextToken().token);
+
+ System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+ System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+ System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+ System.out.println("2nd: " + tokenizer.getNextToken().token);
+
+ }
+
}
\ No newline at end of file