/*
- * Svjatoslav Commons - shared library of common functionality.
- * Copyright ©2012-2017, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of version 3 of the GNU Lesser General Public License
- * or later as published by the Free Software Foundation.
+ * Svjatoslav Commons - shared library of common functionality. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
*/
-
package eu.svjatoslav.commons.string.tokenizer;
import java.util.ArrayList;
public class Tokenizer {
- final Stack<Integer> tokenIndexes = new Stack<>();
+ private final Stack<Integer> tokenIndexes = new Stack<>();
private final List<Terminator> terminators = new ArrayList<>();
private String source;
private int currentIndex = 0;
- int cachedTerminatorIndex = -1;
- Terminator cachedTerminator;
+ private int cachedTerminatorIndex = -1;
+ private Terminator cachedTerminator;
public Tokenizer(final String source) {
this.source = source;
}
- public Tokenizer(){}
+ public Tokenizer() {
+ }
- public Tokenizer setSource(String source){
+ public Tokenizer setSource(String source) {
this.source = source;
currentIndex = 0;
tokenIndexes.clear();
}
-
+ /**
+ *
+ * @return next @TokenizerMatch or <code>null</code> if end of input is reached.
+ * @throws InvalidSyntaxException
+ */
public TokenizerMatch getNextToken() throws InvalidSyntaxException {
tokenIndexes.push(currentIndex);
StringBuilder tokenAccumulator = new StringBuilder();
- while (true){
+ while (true) {
- if (currentIndex >= source.length()){ // reached end of input
+ if (currentIndex >= source.length()) { // reached end of input
if (hasAccumulatedToken(tokenAccumulator))
return new TokenizerMatch(tokenAccumulator.toString(), null, null);
else
if (terminator.termination == PRESERVE)
return buildPreservedToken(tokenAccumulator, terminator);
- else if (terminator.termination == DROP){
+ else if (terminator.termination == DROP) {
skipUntilTerminatorEnd(terminator);
if (hasAccumulatedToken(tokenAccumulator))
currentIndex += terminator.startSequence.length();
}
- private TokenizerMatch buildPreservedToken(StringBuilder token, Terminator terminator) throws InvalidSyntaxException {
+ /**
+ * @throws InvalidSyntaxException if end sequence is not found as is expected by given token.
+ */
+ private TokenizerMatch buildPreservedToken(StringBuilder token, Terminator terminator)
+ throws InvalidSyntaxException {
if (hasAccumulatedToken(token))
return new TokenizerMatch(token.toString(), null, terminator);
if (terminator.hasEndSequence())
- return buildComplexPreservedToken(terminator);
+ return buildTokenWithExpectedENdSequence(terminator);
else
- return buildSimplePreservedToken(terminator);
+ return buildTokenWithoutEndSequence(terminator);
}
- private TokenizerMatch buildSimplePreservedToken(Terminator terminator) {
+ private TokenizerMatch buildTokenWithoutEndSequence(Terminator terminator) {
currentIndex += terminator.startSequence.length();
return new TokenizerMatch(terminator.startSequence, null, terminator);
}
- private TokenizerMatch buildComplexPreservedToken(Terminator terminator) throws InvalidSyntaxException {
+ private TokenizerMatch buildTokenWithExpectedENdSequence(Terminator terminator) throws InvalidSyntaxException {
int endSequenceIndex = getEndSequenceIndex(terminator);
String reminder = source.substring(currentIndex + terminator.startSequence.length(), endSequenceIndex);
currentIndex = endSequenceIndex + terminator.endSequence.length();
return new TokenizerMatch(terminator.startSequence, reminder, terminator);
}
+ /**
+ * @throws InvalidSyntaxException if end of input is reached without finding expected end sequence.
+ */
private int getEndSequenceIndex(Terminator terminator) throws InvalidSyntaxException {
int endSequenceIndex = source.indexOf(terminator.endSequence,
currentIndex + terminator.startSequence.length());
return getOrFindTokenTerminator() == null;
}
- public boolean hasMoreTokens(){
+ public boolean hasMoreTokens() {
return currentIndex < source.length();
}
return result;
}
- public boolean peekIsOneOf(String ... possibilities) throws InvalidSyntaxException {
+ public boolean peekIsOneOf(String... possibilities) throws InvalidSyntaxException {
String nextToken = peekNextToken().token;
return Stream.of(possibilities).anyMatch(possibility -> possibility.equals(nextToken));
}
- public void peekExpectNoneOf(String ... possibilities) throws InvalidSyntaxException {
+ public void peekExpectNoneOf(String... possibilities) throws InvalidSyntaxException {
if (peekIsOneOf(possibilities))
throw new InvalidSyntaxException("Not expected \"" + peekNextToken().token + "\" here.");
}
-
+
public void unreadToken() {
currentIndex = tokenIndexes.pop();
}
+ public void skipUntilDataEnd() {
+ tokenIndexes.push(currentIndex);
+ currentIndex = source.length();
+ }
+
}