/*
* Svjatoslav Commons - shared library of common functionality.
* Copyright ©2012-2017, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu
- *
+ *
* This program is free software; you can redistribute it and/or
* modify it under the terms of version 3 of the GNU Lesser General Public License
* or later as published by the Free Software Foundation.
public class Tokenizer {
- final Stack<Integer> tokenIndexes = new Stack<>();
+ private final Stack<Integer> tokenIndexes = new Stack<>();
private final List<Terminator> terminators = new ArrayList<>();
private String source;
private int currentIndex = 0;
- int cachedTerminatorIndex = -1;
- Terminator cachedTerminator;
+ private int cachedTerminatorIndex = -1;
+ private Terminator cachedTerminator;
public Tokenizer(final String source) {
this.source = source;
}
- public Tokenizer(){}
+ public Tokenizer() {
+ }
- public Tokenizer setSource(String source){
+ public Tokenizer setSource(String source) {
this.source = source;
currentIndex = 0;
tokenIndexes.clear();
}
-
public TokenizerMatch getNextToken() throws InvalidSyntaxException {
tokenIndexes.push(currentIndex);
StringBuilder tokenAccumulator = new StringBuilder();
- while (true){
+ while (true) {
- if (currentIndex >= source.length()){ // reached end of input
+ if (currentIndex >= source.length()) { // reached end of input
if (hasAccumulatedToken(tokenAccumulator))
return new TokenizerMatch(tokenAccumulator.toString(), null, null);
else
if (terminator.termination == PRESERVE)
return buildPreservedToken(tokenAccumulator, terminator);
- else if (terminator.termination == DROP){
+ else if (terminator.termination == DROP) {
skipUntilTerminatorEnd(terminator);
if (hasAccumulatedToken(tokenAccumulator))
return getOrFindTokenTerminator() == null;
}
- public boolean hasMoreTokens(){
+ public boolean hasMoreTokens() {
return currentIndex < source.length();
}
return result;
}
- public boolean peekIsOneOf(String ... possibilities) throws InvalidSyntaxException {
+ public boolean peekIsOneOf(String... possibilities) throws InvalidSyntaxException {
String nextToken = peekNextToken().token;
return Stream.of(possibilities).anyMatch(possibility -> possibility.equals(nextToken));
}
- public void peekExpectNoneOf(String ... possibilities) throws InvalidSyntaxException {
+ public void peekExpectNoneOf(String... possibilities) throws InvalidSyntaxException {
if (peekIsOneOf(possibilities))
throw new InvalidSyntaxException("Not expected \"" + peekNextToken().token + "\" here.");
}
-
+
public void unreadToken() {
currentIndex = tokenIndexes.pop();
}
+ public void skipUntilDataEnd() {
+ tokenIndexes.push(currentIndex);
+ currentIndex = source.length();
+ }
+
}