X-Git-Url: http://www2.svjatoslav.eu/gitweb/?p=svjatoslav_commons.git;a=blobdiff_plain;f=src%2Fmain%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizer.java;h=ba1590db9e8152bf8f64e1072b708b9d960fb10e;hp=b6c0eee3c0d2eb98dbcfd169615a39ae6237da8a;hb=fac4a7627b6496ff15c113adf6eaf2929520c043;hpb=b34ba4499cfbca09bc794a810e460bf1c86dcd34 diff --git a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java index b6c0eee..ba1590d 100755 --- a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java +++ b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java @@ -1,6 +1,6 @@ /* * Svjatoslav Commons - shared library of common functionality. - * Copyright ©2012-2014, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * Copyright ©2012-2017, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu * * This program is free software; you can redistribute it and/or * modify it under the terms of version 3 of the GNU Lesser General Public License @@ -13,117 +13,143 @@ import java.util.ArrayList; import java.util.List; import java.util.Stack; +import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP; + public class Tokenizer { - private final List terminators = new ArrayList(); - private final String source; - - Stack tokenIndexes = new Stack(); - - private int currentIndex = 0; - - public Tokenizer(final String source) { - this.source = source; - } - - public void addTerminator(final String startSequence, - final boolean ignoreTerminator) { - terminators.add(new Terminator(startSequence, ignoreTerminator)); - } - - public void addTerminator(final String startSequence, - final String endSequence, final boolean ignoreTerminator) { - terminators.add(new Terminator(startSequence, endSequence, - ignoreTerminator)); - } - - public void expectNextToken(final String value) - throws InvalidSyntaxException { - final TokenizerMatch match = getNextToken(); - if (!value.equals(match.token)) - throw new InvalidSyntaxException("Expected \"" + value - + "\" but got \"" + match.token + "\" instead."); - } - - public TokenizerMatch getNextToken() { - tokenIndexes.push(currentIndex); - final StringBuffer result = new StringBuffer(); - - while (true) { - if (currentIndex >= source.length()) - return null; - - boolean accumulateCurrentChar = true; - - findTerminator: for (final Terminator terminator : terminators) - if (sequenceMatches(terminator.startSequence)) - - if (terminator.ignoreTerminator) { - currentIndex += terminator.startSequence.length(); - - if (terminator.endSequence != null) - skipUntilSequence(terminator.endSequence); - - if (result.length() > 0) - return new TokenizerMatch(result.toString(), - terminator); - else { - accumulateCurrentChar = false; - break findTerminator; - } - } else if (result.length() > 0) - return new TokenizerMatch(result.toString(), terminator); - else { - currentIndex += terminator.startSequence.length(); - return new TokenizerMatch(terminator.startSequence, - terminator); - } - - if (accumulateCurrentChar) { - result.append(source.charAt(currentIndex)); - currentIndex++; - } - } - - } - - public boolean probeNextToken(final String token) { - if (token.equals(getNextToken().token)) - return true; - - unreadToken(); - return false; - } - - public boolean sequenceMatches(final String sequence) { - if ((currentIndex + sequence.length()) > source.length()) - return false; - - for (int i = 0; i < sequence.length(); i++) - if (sequence.charAt(i) != source.charAt(i + currentIndex)) - return false; - - return true; - } - - public void skipUntilDataEnd() { - tokenIndexes.push(currentIndex); - currentIndex = source.length(); - } - - public void skipUntilSequence(final String sequence) { - while (currentIndex < source.length()) { - if (sequenceMatches(sequence)) { - currentIndex += sequence.length(); - return; - } - - currentIndex++; - } - } - - public void unreadToken() { - currentIndex = tokenIndexes.pop(); - } + final Stack tokenIndexes = new Stack<>(); + private final List terminators = new ArrayList<>(); + private String source; + private int currentIndex = 0; + + public Tokenizer(final String source) { + this.source = source; + } + + public Tokenizer(){} + + public Tokenizer setSource(String source){ + this.source = source; + currentIndex = 0; + return this; + } + + public Tokenizer addTerminator(final String startSequence, + final Terminator.TerminationStrategy terminationStrategy) { + terminators.add(new Terminator(startSequence, terminationStrategy)); + return this; + } + + public Tokenizer addTerminator(final String startSequence, + final String endSequence, final Terminator.TerminationStrategy terminationStrategy) { + terminators.add(new Terminator(startSequence, endSequence, terminationStrategy)); + return this; + } + + public void expectNextToken(final String value) + throws InvalidSyntaxException { + final TokenizerMatch match = getNextToken(); + if (!value.equals(match.token)) + throw new InvalidSyntaxException("Expected \"" + value + + "\" but got \"" + match.token + "\" instead."); + } + + public TokenizerMatch getNextToken() { + tokenIndexes.push(currentIndex); + final StringBuilder result = new StringBuilder(); + + while (true) { + if (currentIndex >= source.length()) + return null; + + boolean accumulateCurrentChar = true; + + for (final Terminator terminator : terminators) + if (sequenceMatches(terminator.startSequence)) + + if (terminator.termination == DROP) { + currentIndex += terminator.startSequence.length(); + + if (terminator.endSequence != null) + skipUntilSequence(terminator.endSequence); + + if (result.length() > 0) + return new TokenizerMatch(result.toString(), + terminator); + else { + accumulateCurrentChar = false; + break; + } + } else if (result.length() > 0) + return new TokenizerMatch(result.toString(), terminator); + else { + currentIndex += terminator.startSequence.length(); + return new TokenizerMatch(terminator.startSequence, + terminator); + } + + if (accumulateCurrentChar) { + result.append(source.charAt(currentIndex)); + currentIndex++; + } + } + + } + + public boolean consumeIfNextToken(final String token) { + if (token.equals(getNextToken().token)) + return true; + + unreadToken(); + return false; + } + + public TokenizerMatch peekNextToken(){ + TokenizerMatch result = getNextToken(); + unreadToken(); + return result; + } + + public boolean peekIsOneOf(String [] ... possibilities){ + TokenizerMatch nextToken = peekNextToken(); + + for (String[] possibility : possibilities) + if (possibility.equals(nextToken)) + return true; + + return false; + } + + + public boolean sequenceMatches(final String sequence) { + if ((currentIndex + sequence.length()) > source.length()) + return false; + + for (int i = 0; i < sequence.length(); i++) + if (sequence.charAt(i) != source.charAt(i + currentIndex)) + return false; + + return true; + } + + public void skipUntilDataEnd() { + tokenIndexes.push(currentIndex); + currentIndex = source.length(); + } + + public void skipUntilSequence(final String sequence) { + while (currentIndex < source.length()) { + if (sequenceMatches(sequence)) { + currentIndex += sequence.length(); + return; + } + + currentIndex++; + } + } + + public void unreadToken() { + currentIndex = tokenIndexes.pop(); + } }