X-Git-Url: http://www2.svjatoslav.eu/gitweb/?p=svjatoslav_commons.git;a=blobdiff_plain;f=src%2Fmain%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizer.java;h=b716989269dba2d37c586d17cda890b7de39ea1b;hp=4eb9dde36ff4679a5f2280f486ec893ed55c1d4b;hb=37f0e9901f9ecaea7a28cba616e247415ebb9ea0;hpb=b46b1b0c0ded3377fb44d88fe4105ffebc8ee152

diff --git a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
index 4eb9dde..b716989 100755
--- a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
+++ b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
@@ -60,6 +60,11 @@ public class Tokenizer {
     }
 
 
+    /**
+     *
+     * @return next @TokenizerMatch or <code>null</code> if end of input is reached.
+     * @throws InvalidSyntaxException
+     */
     public TokenizerMatch getNextToken() throws InvalidSyntaxException {
         tokenIndexes.push(currentIndex);
 
@@ -101,22 +106,26 @@ public class Tokenizer {
             currentIndex += terminator.startSequence.length();
     }
 
-    private TokenizerMatch buildPreservedToken(StringBuilder token, Terminator terminator) throws InvalidSyntaxException {
+    /**
+     * @throws InvalidSyntaxException if end sequence is not found as is expected by given token.
+     */
+    private TokenizerMatch buildPreservedToken(StringBuilder token, Terminator terminator)
+            throws InvalidSyntaxException {
         if (hasAccumulatedToken(token))
             return new TokenizerMatch(token.toString(), null, terminator);
 
         if (terminator.hasEndSequence())
-            return buildComplexPreservedToken(terminator);
+            return buildTokenWithExpectedENdSequence(terminator);
         else
-            return buildSimplePreservedToken(terminator);
+            return buildTokenWithoutEndSequence(terminator);
     }
 
-    private TokenizerMatch buildSimplePreservedToken(Terminator terminator) {
+    private TokenizerMatch buildTokenWithoutEndSequence(Terminator terminator) {
         currentIndex += terminator.startSequence.length();
         return new TokenizerMatch(terminator.startSequence, null, terminator);
     }
 
-    private TokenizerMatch buildComplexPreservedToken(Terminator terminator) throws InvalidSyntaxException {
+    private TokenizerMatch buildTokenWithExpectedENdSequence(Terminator terminator) throws InvalidSyntaxException {
         int endSequenceIndex = getEndSequenceIndex(terminator);
         String reminder = source.substring(currentIndex + terminator.startSequence.length(), endSequenceIndex);
         currentIndex = endSequenceIndex + terminator.endSequence.length();
@@ -124,6 +133,9 @@ public class Tokenizer {
         return new TokenizerMatch(terminator.startSequence, reminder, terminator);
     }
 
+    /**
+     * @throws InvalidSyntaxException if end of input is reached without finding expected end sequence.
+     */
     private int getEndSequenceIndex(Terminator terminator) throws InvalidSyntaxException {
         int endSequenceIndex = source.indexOf(terminator.endSequence,
                 currentIndex + terminator.startSequence.length());