src/test/java/eu/svjatoslav/commons/string/tokenizer/TokenizerTest.java

   1 package eu.svjatoslav.commons.string.tokenizer;
   2
   3 import org.junit.Test;
   4
   5 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
   6 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
   7 import static org.junit.Assert.assertEquals;
   8
   9 public class TokenizerTest {
  10     @Test
  11     public void findTokenTerminator() throws Exception {
  12
  13         Tokenizer tokenizer = new Tokenizer("this /* comment */ a test")
  14                 .addTerminator("/*", "*/", PRESERVE);
  15
  16
  17
  18     }
  19
  20
  21     @Test
  22     public void you_can_peek() throws Exception {
  23         Tokenizer tokenizer = new Tokenizer("this is a N'2015-03-18 09:48:54.360' test")
  24                 .addTerminator(" ", DROP)
  25                 .addTerminator("N'", "'", PRESERVE);
  26
  27         tokenizer.expectAndConsumeNextToken("this");
  28
  29         assertEquals("is", tokenizer.peekNextToken().token);
  30
  31         assertEquals("is", tokenizer.peekNextToken().token);
  32
  33         assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
  34     }
  35
  36     @Test
  37     public void complexTerminator() throws Exception {
  38         Tokenizer tokenizer = new Tokenizer("/* hello */ /** comment **/   ((  is a N'2015-03-18 09:48:54.360' test")
  39                 .addTerminator(" ", DROP)
  40                 .addTerminator("(", PRESERVE)
  41                 .addTerminator("\"", "\"" , PRESERVE)
  42                 .addTerminator("/*", "*/" , PRESERVE)
  43                 ;
  44
  45         TokenizerMatch nextToken = tokenizer.getNextToken();
  46         System.out.println("T: \"" + nextToken.token + "\", R: \"" + nextToken.reminder + "\"");
  47         System.out.println(tokenizer.getNextToken().token);
  48         System.out.println(tokenizer.getNextToken().token);
  49         System.out.println(tokenizer.getNextToken().token);
  50         System.out.println(tokenizer.getNextToken().token);
  51         System.out.println(tokenizer.getNextToken().token);
  52         System.out.println(tokenizer.getNextToken().token);
  53         System.out.println(tokenizer.getNextToken().token);
  54         System.out.println(tokenizer.getNextToken().token);
  55
  56
  57 //        tokenizer.expectAndConsumeNextToken("this");
  58 //
  59 //        assertEquals("is", tokenizer.peekNextToken().token);
  60 //
  61 //        assertEquals("is", tokenizer.peekNextToken().token);
  62 //
  63 //        assertEquals(true, tokenizer.peekIsOneOf("maybe", "is", "that"));
  64     }
  65
  66
  67     @Test
  68     public void testComplexTerminator() throws Exception {
  69         Tokenizer tokenizer = new Tokenizer("this N'2015-03-18 09:48:54.360'  /* thoe unto u */ test")
  70                 .addTerminator(" ", DROP)
  71                 .addTerminator("/*", "*/", PRESERVE);
  72
  73 //        tokenizer.expectAndConsumeNextToken("this");
  74
  75 //        assertEquals("2015-03-18 09:48:54.360", tokenizer.getNextToken().token);
  76
  77         System.out.println("1st: " + tokenizer.getNextToken().token);
  78
  79         System.out.println("2nd: " + tokenizer.getNextToken().token);
  80
  81         System.out.println("2nd: " + tokenizer.getNextToken().token);
  82
  83         System.out.println("2nd: " + tokenizer.getNextToken().token);
  84
  85         System.out.println("2nd: " + tokenizer.getNextToken().token);
  86
  87     }
  88
  89 }