X-Git-Url: http://www2.svjatoslav.eu/gitweb/?a=blobdiff_plain;f=src%2Ftest%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizerTest.java;h=9f3536799081c21b8fbb7e672e9947573fe7b721;hb=67f7af91a79bc2ff50071389b6333a28755a4bff;hp=ddb2662bfd8c81022313135f3889ad09eb42c820;hpb=3bc3db3ceb288b82e48c349bf27dfafda2bcd444;p=svjatoslav_commons.git diff --git a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java index ddb2662..9f35367 100644 --- a/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java +++ b/src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java @@ -4,22 +4,12 @@ import org.junit.Test; import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP; import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; public class TokenizerTest { - @Test - public void findTokenTerminator() throws Exception { - - Tokenizer tokenizer = new Tokenizer("this /* comment */ a test") - .addTerminator("/*", "*/", PRESERVE); - - - - } - @Test - public void you_can_peek() throws Exception { + public void testPeeking() throws Exception { Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test") .addTerminator(" ", DROP) .addTerminator("N'", "'", PRESERVE); @@ -34,56 +24,43 @@ public class TokenizerTest { } @Test - public void complexTerminator() throws Exception { - Tokenizer tokenizer = new Tokenizer("/* hello */ /** comment **/ (( is a N'2015-03-18 09:48:54.360' test") + public void testTokenization() throws Exception { + Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/ (( is a N'2015-03-18 09:48:54.360' test") .addTerminator(" ", DROP) .addTerminator("(", PRESERVE) - .addTerminator("\"", "\"" , PRESERVE) - .addTerminator("/*", "*/" , PRESERVE) - ; - - TokenizerMatch nextToken = tokenizer.getNextToken(); - System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\""); - System.out.println(tokenizer.getNextToken().token); - System.out.println(tokenizer.getNextToken().token); - System.out.println(tokenizer.getNextToken().token); - System.out.println(tokenizer.getNextToken().token); - System.out.println(tokenizer.getNextToken().token); - System.out.println(tokenizer.getNextToken().token); - System.out.println(tokenizer.getNextToken().token); - System.out.println(tokenizer.getNextToken().token); - - -// tokenizer.expectAndConsumeNextToken("this"); -// -// assertEquals("is", tokenizer.peekNextToken().token); -// -// assertEquals("is", tokenizer.peekNextToken().token); -// -// assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that")); + .addTerminator("\"", "\"", PRESERVE) + .addTerminator("N'", "'", PRESERVE) + .addTerminator("/*", "*/", DROP); + + assertTokenEquals("\"", "hello", tokenizer); + assertTokenEquals("(", null, tokenizer); + assertTokenEquals("(", null, tokenizer); + assertTokenEquals("is", null, tokenizer); + assertTokenEquals("a", null, tokenizer); + assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer); + assertTokenEquals("test", null, tokenizer); + + assertNull(tokenizer.getNextToken()); + assertFalse(tokenizer.hasMoreContent()); } + private void assertTokenEquals(String token, String reminder, Tokenizer tokenizer) throws InvalidSyntaxException { + TokenizerMatch nextToken = tokenizer.getNextToken(); - @Test - public void testComplexTerminator() throws Exception { - Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360' /* thoe unto u */ test") - .addTerminator(" ", DROP) - .addTerminator("/*", "*/", PRESERVE); - -// tokenizer.expectAndConsumeNextToken("this"); - -// assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token); - - System.out.println("1st: " + tokenizer.getNextToken().token); - - System.out.println("2nd: " + tokenizer.getNextToken().token); - - System.out.println("2nd: " + tokenizer.getNextToken().token); - - System.out.println("2nd: " + tokenizer.getNextToken().token); + assertEquals(token, nextToken.token); - System.out.println("2nd: " + tokenizer.getNextToken().token); + if (reminder == null) + assertNull(nextToken.reminder); + else + assertEquals(reminder, nextToken.reminder); + } + private void debugNextToken(Tokenizer tokenizer) throws InvalidSyntaxException { + TokenizerMatch nextToken = tokenizer.getNextToken(); + if (nextToken == null) + System.out.println("null"); + else + System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\""); } } \ No newline at end of file