import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
public class TokenizerTest {
- @Test
- public void findTokenTerminator() throws Exception {
-
- Tokenizer tokenizer = new Tokenizer("this /* comment */ a test")
- .addTerminator("/*", "*/", PRESERVE);
-
-
-
- }
-
@Test
- public void you_can_peek() throws Exception {
+ public void testPeeking() throws Exception {
Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
.addTerminator(" ", DROP)
.addTerminator("N'", "'", PRESERVE);
}
@Test
- public void complexTerminator() throws Exception {
- Tokenizer tokenizer = new Tokenizer("/* hello */ /** comment **/ (( is a N'2015-03-18 09:48:54.360' test")
+ public void testTokenization() throws Exception {
+ Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/ (( is a N'2015-03-18 09:48:54.360' test")
.addTerminator(" ", DROP)
.addTerminator("(", PRESERVE)
.addTerminator("\"", "\"" , PRESERVE)
- .addTerminator("/*", "*/" , PRESERVE)
+ .addTerminator("N'", "'" , PRESERVE)
+ .addTerminator("/*", "*/" , DROP)
;
- TokenizerMatch nextToken = tokenizer.getNextToken();
- System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
- System.out.println(tokenizer.getNextToken().token);
-
+ assertTokenEquals("\"", "hello", tokenizer);
+ assertTokenEquals("(", null, tokenizer);
+ assertTokenEquals("(", null, tokenizer);
+ assertTokenEquals("is", null, tokenizer);
+ assertTokenEquals("a", null, tokenizer);
+ assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
+ assertTokenEquals("test", null, tokenizer);
-// tokenizer.expectAndConsumeNextToken("this");
-//
-// assertEquals("is", tokenizer.peekNextToken().token);
-//
-// assertEquals("is", tokenizer.peekNextToken().token);
-//
-// assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
+ assertNull(tokenizer.getNextToken());
+ assertFalse(tokenizer.hasMoreTokens());
}
+ private void assertTokenEquals(String token, String reminder, Tokenizer tokenizer) throws InvalidSyntaxException {
+ TokenizerMatch nextToken = tokenizer.getNextToken();
- @Test
- public void testComplexTerminator() throws Exception {
- Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360' /* thoe unto u */ test")
- .addTerminator(" ", DROP)
- .addTerminator("/*", "*/", PRESERVE);
-
-// tokenizer.expectAndConsumeNextToken("this");
-
-// assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
-
- System.out.println("1st: " + tokenizer.getNextToken().token);
-
- System.out.println("2nd: " + tokenizer.getNextToken().token);
-
- System.out.println("2nd: " + tokenizer.getNextToken().token);
-
- System.out.println("2nd: " + tokenizer.getNextToken().token);
-
- System.out.println("2nd: " + tokenizer.getNextToken().token);
+ assertEquals(token, nextToken.token);
+ if (reminder == null)
+ assertNull(nextToken.reminder);
+ else
+ assertEquals(reminder, nextToken.reminder);
}
+ private void debugNextToken(Tokenizer tokenizer) throws InvalidSyntaxException {
+ TokenizerMatch nextToken = tokenizer.getNextToken();
+ if (nextToken == null)
+ System.out.println("null");
+ else
+ System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
+ }
+
}
\ No newline at end of file