src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java

   1 package eu.svjatoslav.commons.string.tokenizer;
   2
   3 import org.junit.Test;
   4
   5 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
   6 import static org.junit.Assert.assertEquals;
   7
   8 public class TokenizerTest {
   9     @Test
  10     public void findTokenTerminator() throws Exception {
  11
  12         Tokenizer tokenizer = new Tokenizer("this /* comment */ a test")
  13                 .addTerminator("/*", "*/", PRESERVE);
  14
  15
  16
  17     }
  18
  19
  20     @Test
  21     public void you_can_peek() throws Exception {
  22         Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
  23                 .addTerminator(" ", Terminator.TerminationStrategy.DROP)
  24                 .addTerminator("N'", "'", PRESERVE);
  25
  26         tokenizer.expectAndConsumeNextToken("this");
  27
  28         assertEquals("is", tokenizer.peekNextToken().token);
  29
  30         assertEquals("is", tokenizer.peekNextToken().token);
  31
  32         assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
  33     }
  34
  35     @Test
  36     public void complexTerminator() throws Exception {
  37         Tokenizer tokenizer = new Tokenizer("   this((\"hello\"  /* comment */   ((  is a N'2015-03-18 09:48:54.360' test")
  38                 .addTerminator(" ", Terminator.TerminationStrategy.DROP)
  39                 .addTerminator("(", Terminator.TerminationStrategy.PRESERVE)
  40                 .addTerminator("\"", "\"" ,Terminator.TerminationStrategy.PRESERVE)
  41                 .addTerminator("/*", "*/" ,Terminator.TerminationStrategy.DROP)
  42                 ;
  43
  44         System.out.println(tokenizer.getNextToken().token);
  45         System.out.println(tokenizer.getNextToken().token);
  46         System.out.println(tokenizer.getNextToken().token);
  47         System.out.println(tokenizer.getNextToken().token);
  48         System.out.println(tokenizer.getNextToken().token);
  49         System.out.println(tokenizer.getNextToken().token);
  50         System.out.println(tokenizer.getNextToken().token);
  51         System.out.println(tokenizer.getNextToken().token);
  52         System.out.println(tokenizer.getNextToken().token);
  53
  54
  55 //        tokenizer.expectAndConsumeNextToken("this");
  56 //
  57 //        assertEquals("is", tokenizer.peekNextToken().token);
  58 //
  59 //        assertEquals("is", tokenizer.peekNextToken().token);
  60 //
  61 //        assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
  62     }
  63
  64
  65     @Test
  66     public void testComplexTerminator() throws Exception {
  67         Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360'  /* thoe unto u */ test")
  68                 .addTerminator(" ", Terminator.TerminationStrategy.DROP)
  69                 .addTerminator("/*", "*/", PRESERVE);
  70
  71 //        tokenizer.expectAndConsumeNextToken("this");
  72
  73 //        assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
  74
  75         System.out.println("1st: " + tokenizer.getNextToken().token);
  76
  77         System.out.println("2nd: " + tokenizer.getNextToken().token);
  78
  79         System.out.println("2nd: " + tokenizer.getNextToken().token);
  80
  81         System.out.println("2nd: " + tokenizer.getNextToken().token);
  82
  83         System.out.println("2nd: " + tokenizer.getNextToken().token);
  84
  85     }
  86
  87 }