Added clarifying comments

diff --git a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Terminator.java b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Terminator.java
index f628d2b..1a6c5ee 100755 (executable)
--- a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Terminator.java
+++ b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Terminator.java
@@ -49,7 +49,7 @@ public class Terminator {
     }
 
     public enum TerminationStrategy {
-        PRESERVE,
-        DROP
+        PRESERVE, // Identify and return such tokens for further processing.
+        DROP // Identify but ignore such tokens, do not return them. Good for handling comments in scripts.
     }
 }

diff --git a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
index 4eb9dde..b716989 100755 (executable)
--- a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
+++ b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
@@ -60,6 +60,11 @@ public class Tokenizer {
     }
 
 
+    /**
+     *
+     * @return next @TokenizerMatch or <code>null</code> if end of input is reached.
+     * @throws InvalidSyntaxException
+     */
     public TokenizerMatch getNextToken() throws InvalidSyntaxException {
         tokenIndexes.push(currentIndex);
 
@@ -101,22 +106,26 @@ public class Tokenizer {
             currentIndex += terminator.startSequence.length();
     }
 
-    private TokenizerMatch buildPreservedToken(StringBuilder token, Terminator terminator) throws InvalidSyntaxException {
+    /**
+     * @throws InvalidSyntaxException if end sequence is not found as is expected by given token.
+     */
+    private TokenizerMatch buildPreservedToken(StringBuilder token, Terminator terminator)
+            throws InvalidSyntaxException {
         if (hasAccumulatedToken(token))
             return new TokenizerMatch(token.toString(), null, terminator);
 
         if (terminator.hasEndSequence())
-            return buildComplexPreservedToken(terminator);
+            return buildTokenWithExpectedENdSequence(terminator);
         else
-            return buildSimplePreservedToken(terminator);
+            return buildTokenWithoutEndSequence(terminator);
     }
 
-    private TokenizerMatch buildSimplePreservedToken(Terminator terminator) {
+    private TokenizerMatch buildTokenWithoutEndSequence(Terminator terminator) {
         currentIndex += terminator.startSequence.length();
         return new TokenizerMatch(terminator.startSequence, null, terminator);
     }
 
-    private TokenizerMatch buildComplexPreservedToken(Terminator terminator) throws InvalidSyntaxException {
+    private TokenizerMatch buildTokenWithExpectedENdSequence(Terminator terminator) throws InvalidSyntaxException {
         int endSequenceIndex = getEndSequenceIndex(terminator);
         String reminder = source.substring(currentIndex + terminator.startSequence.length(), endSequenceIndex);
         currentIndex = endSequenceIndex + terminator.endSequence.length();
@@ -124,6 +133,9 @@ public class Tokenizer {
         return new TokenizerMatch(terminator.startSequence, reminder, terminator);
     }
 
+    /**
+     * @throws InvalidSyntaxException if end of input is reached without finding expected end sequence.
+     */
     private int getEndSequenceIndex(Terminator terminator) throws InvalidSyntaxException {
         int endSequenceIndex = source.indexOf(terminator.endSequence,
                 currentIndex + terminator.startSequence.length());