src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java

   1 package eu.svjatoslav.commons.string.tokenizer;
   2
   3 import org.junit.Test;
   4
   5 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
   6 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
   7 import static org.junit.Assert.*;
   8
   9 public class TokenizerTest {
  10
  11     @Test
  12     public void testPeeking() throws Exception {
  13         Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
  14                 .addTerminator(" ", DROP)
  15                 .addTerminator("N'", "'", PRESERVE);
  16
  17         tokenizer.expectAndConsumeNextToken("this");
  18
  19         assertEquals("is", tokenizer.peekNextToken().token);
  20
  21         assertEquals("is", tokenizer.peekNextToken().token);
  22
  23         assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
  24     }
  25
  26     @Test
  27     public void testTokenization() throws Exception {
  28         Tokenizer tokenizer = new Tokenizer("\"hello\" /** comment **/   ((  is a N'2015-03-18 09:48:54.360' test")
  29                 .addTerminator(" ", DROP)
  30                 .addTerminator("(", PRESERVE)
  31                 .addTerminator("\"", "\"", PRESERVE)
  32                 .addTerminator("N'", "'", PRESERVE)
  33                 .addTerminator("/*", "*/", DROP);
  34
  35         assertTokenEquals("\"", "hello", tokenizer);
  36         assertTokenEquals("(", null, tokenizer);
  37         assertTokenEquals("(", null, tokenizer);
  38         assertTokenEquals("is", null, tokenizer);
  39         assertTokenEquals("a", null, tokenizer);
  40         assertTokenEquals("N'", "2015-03-18 09:48:54.360", tokenizer);
  41         assertTokenEquals("test", null, tokenizer);
  42
  43         assertNull(tokenizer.getNextToken());
  44         assertFalse(tokenizer.hasMoreContent());
  45     }
  46
  47     private void assertTokenEquals(String token, String reminder, Tokenizer tokenizer) throws InvalidSyntaxException {
  48         TokenizerMatch nextToken = tokenizer.getNextToken();
  49
  50         assertEquals(token, nextToken.token);
  51
  52         if (reminder == null)
  53             assertNull(nextToken.reminder);
  54         else
  55             assertEquals(reminder, nextToken.reminder);
  56     }
  57
  58     private void debugNextToken(Tokenizer tokenizer) throws InvalidSyntaxException {
  59         TokenizerMatch nextToken = tokenizer.getNextToken();
  60         if (nextToken == null)
  61             System.out.println("null");
  62         else
  63             System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
  64     }
  65
  66 }