src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java

   1 package eu.svjatoslav.commons.string.tokenizer;
   2
   3 import org.junit.Test;
   4
   5 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
   6 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
   7 import static org.junit.Assert.assertEquals;
   8 import static org.junit.Assert.assertFalse;
   9 import static org.junit.Assert.assertNull;
  10
  11 public class TokenizerTest {
  12
  13     @Test
  14     public void testPeeking() throws Exception {
  15         Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
  16                 .addTerminator(" ", DROP)
  17                 .addTerminator("N'", "'", PRESERVE);
  18
  19         tokenizer.expectAndConsumeNextToken("this");
  20
  21         assertEquals("is", tokenizer.peekNextToken().token);
  22
  23         assertEquals("is", tokenizer.peekNextToken().token);
  24
  25         assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
  26     }
  27
  28     @Test
  29     public void testTokenization() throws Exception {
  30         Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/   ((  is a N'2015-03-18 09:48:54.360' test")
  31                 .addTerminator(" ", DROP)
  32                 .addTerminator("(", PRESERVE)
  33                 .addTerminator("\"", "\"" , PRESERVE)
  34                 .addTerminator("N'", "'" , PRESERVE)
  35                 .addTerminator("/*", "*/" , DROP)
  36                 ;
  37
  38         assertTokenEquals("\"", "hello", tokenizer);
  39         assertTokenEquals("(", null, tokenizer);
  40         assertTokenEquals("(", null, tokenizer);
  41         assertTokenEquals("is", null, tokenizer);
  42         assertTokenEquals("a", null, tokenizer);
  43         assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
  44         assertTokenEquals("test", null, tokenizer);
  45
  46         assertNull(tokenizer.getNextToken());
  47         assertFalse(tokenizer.hasMoreTokens());
  48     }
  49
  50     private void assertTokenEquals(String token, String reminder, Tokenizer tokenizer) throws InvalidSyntaxException {
  51         TokenizerMatch nextToken = tokenizer.getNextToken();
  52
  53         assertEquals(token, nextToken.token);
  54
  55         if (reminder == null)
  56             assertNull(nextToken.reminder);
  57         else
  58             assertEquals(reminder, nextToken.reminder);
  59     }
  60
  61     private void debugNextToken(Tokenizer tokenizer) throws InvalidSyntaxException {
  62         TokenizerMatch nextToken = tokenizer.getNextToken();
  63         if (nextToken == null)
  64             System.out.println("null");
  65         else
  66             System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
  67     }
  68
  69 }