X-Git-Url: http://www2.svjatoslav.eu/gitweb/?p=svjatoslav_commons.git;a=blobdiff_plain;f=src%2Fmain%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizer.java;h=14554beb886acffc4e638b31b5e50c49cb6ada51;hp=140773218e9fdb448c091b54ab8fd562662f8bdf;hb=b1ffc7025cc976821987469570f07a7298ea16c9;hpb=b8bd1e820265fc15c39c1ee8c06289ea8b8e2c1c diff --git a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java index 1407732..14554be 100755 --- a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java +++ b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java @@ -1,31 +1,33 @@ /* - * Svjatoslav Commons - shared library of common functionality. - * Copyright ©2012-2017, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of version 3 of the GNU Lesser General Public License - * or later as published by the Free Software Foundation. + * Svjatoslav Commons - shared library of common functionality. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. */ - package eu.svjatoslav.commons.string.tokenizer; import java.util.ArrayList; import java.util.List; import java.util.Stack; +import java.util.regex.Matcher; import java.util.stream.Stream; -import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP; import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE; +import static java.lang.System.out; public class Tokenizer { + /** + * Stack of token indexes. This allows to walk back in history and un-consume the token. + */ private final Stack tokenIndexes = new Stack<>(); + + /** + * Terminators that will be searched for by given tokenizer within given source string. + */ private final List terminators = new ArrayList<>(); - private String source; - private int currentIndex = 0; - private int cachedTerminatorIndex = -1; - private Terminator cachedTerminator; + private String source; // string to be tokenized + + private int currentIndex = 0; public Tokenizer(final String source) { this.source = source; @@ -38,25 +40,29 @@ public class Tokenizer { this.source = source; currentIndex = 0; tokenIndexes.clear(); - - cachedTerminatorIndex = -1; - cachedTerminator = null; return this; } - public Tokenizer addTerminator(final String startSequence, - final Terminator.TerminationStrategy terminationStrategy) { - terminators.add(new Terminator(startSequence, terminationStrategy)); - return this; + public Terminator addTerminator(final Terminator.TerminationStrategy terminationStrategy, String regexp) { + Terminator terminator = new Terminator(terminationStrategy, regexp,null); + terminators.add(terminator); + return terminator; } - public Tokenizer addTerminator(final String startSequence, - final String endSequence, final Terminator.TerminationStrategy terminationStrategy) { - terminators.add(new Terminator(startSequence, endSequence, terminationStrategy)); - return this; + public Terminator addTerminator(final Terminator.TerminationStrategy terminationStrategy, + String regexp, String group) { + Terminator terminator = new Terminator(terminationStrategy, regexp,group); + terminators.add(terminator); + return terminator; + } + + + public Terminator addTerminator(Terminator terminator) { + terminators.add(terminator); + return terminator; } - public void expectAndConsumeNextToken(final String value) + public void expectAndConsumeNextStringToken(final String value) throws InvalidSyntaxException { final TokenizerMatch match = getNextToken(); if (!value.equals(match.token)) @@ -64,8 +70,22 @@ public class Tokenizer { + "\" but got \"" + match.token + "\" instead."); } + public TokenizerMatch expectAndConsumeNextTerminatorToken(Terminator terminator) + throws InvalidSyntaxException { + final TokenizerMatch match = getNextToken(); + + if (match.terminator != terminator) + throw new InvalidSyntaxException("Expected terminator \"" + terminator + + "\" but got \"" + match.terminator + "\" instead."); + + return match; + } + - public TokenizerMatch getNextToken() throws InvalidSyntaxException { + /** + * @return next @TokenizerMatch or null if end of input is reached. + */ + public TokenizerMatch getNextToken() { tokenIndexes.push(currentIndex); StringBuilder tokenAccumulator = new StringBuilder(); @@ -74,102 +94,54 @@ public class Tokenizer { if (currentIndex >= source.length()) { // reached end of input if (hasAccumulatedToken(tokenAccumulator)) - return new TokenizerMatch(tokenAccumulator.toString(), null, null); + return new TokenizerMatch(tokenAccumulator.toString(), null, null, this); else return null; } - if (isOngoingToken()) { + TokenizerMatch matchResult = findTerminatorMatch(); + if (matchResult == null) { tokenAccumulator.append(source.charAt(currentIndex)); currentIndex++; continue; } - Terminator terminator = getOrFindTokenTerminator(); + if (matchResult.terminator.termination == PRESERVE) { + if (hasAccumulatedToken(tokenAccumulator)) + return new TokenizerMatch(tokenAccumulator.toString(), null, null, this); - if (terminator.termination == PRESERVE) - return buildPreservedToken(tokenAccumulator, terminator); - else if (terminator.termination == DROP) { - skipUntilTerminatorEnd(terminator); + currentIndex = matchResult.matcher.end(); + return matchResult; + } else { + currentIndex = matchResult.matcher.end(); if (hasAccumulatedToken(tokenAccumulator)) - return new TokenizerMatch(tokenAccumulator.toString(), null, terminator); + return new TokenizerMatch(tokenAccumulator.toString(), null, null, this); } } - - } - - private void skipUntilTerminatorEnd(Terminator terminator) throws InvalidSyntaxException { - if (terminator.hasEndSequence()) - currentIndex = getEndSequenceIndex(terminator) + terminator.endSequence.length(); - else - currentIndex += terminator.startSequence.length(); - } - - private TokenizerMatch buildPreservedToken(StringBuilder token, Terminator terminator) throws InvalidSyntaxException { - if (hasAccumulatedToken(token)) - return new TokenizerMatch(token.toString(), null, terminator); - - if (terminator.hasEndSequence()) - return buildComplexPreservedToken(terminator); - else - return buildSimplePreservedToken(terminator); } - private TokenizerMatch buildSimplePreservedToken(Terminator terminator) { - currentIndex += terminator.startSequence.length(); - return new TokenizerMatch(terminator.startSequence, null, terminator); - } - - private TokenizerMatch buildComplexPreservedToken(Terminator terminator) throws InvalidSyntaxException { - int endSequenceIndex = getEndSequenceIndex(terminator); - String reminder = source.substring(currentIndex + terminator.startSequence.length(), endSequenceIndex); - currentIndex = endSequenceIndex + terminator.endSequence.length(); - - return new TokenizerMatch(terminator.startSequence, reminder, terminator); - } - - private int getEndSequenceIndex(Terminator terminator) throws InvalidSyntaxException { - int endSequenceIndex = source.indexOf(terminator.endSequence, - currentIndex + terminator.startSequence.length()); - - if (endSequenceIndex < 0) - throw new InvalidSyntaxException("Expected \"" + terminator.endSequence + "\" but not found."); - - return endSequenceIndex; - } - - private boolean hasAccumulatedToken(StringBuilder token) { - return token.length() > 0; + public TokenizerMatch findTerminatorMatch(){ + for (Terminator terminator : terminators) + if (terminator.active) { + Matcher match = terminator.match(source, currentIndex); + if (match.find()) { + String token = source.substring(match.start(), match.end()); + return new TokenizerMatch(token, terminator, match, this); + } + } + return null; } - private boolean isOngoingToken() { - return getOrFindTokenTerminator() == null; + private boolean hasAccumulatedToken(StringBuilder tokenAccumulator) { + return tokenAccumulator.length() > 0; } - public boolean hasMoreTokens() { + public boolean hasMoreContent() { + if (source == null) return false; return currentIndex < source.length(); } - /** - * Attempts to cache terminator search result. - */ - public Terminator getOrFindTokenTerminator() { - if (currentIndex == cachedTerminatorIndex) - return cachedTerminator; - - cachedTerminatorIndex = currentIndex; - cachedTerminator = findTokenTerminator(); - return cachedTerminator; - } - - private Terminator findTokenTerminator() { - for (Terminator terminator : terminators) - if (terminator.matches(source, currentIndex)) - return terminator; - return null; - } - public boolean consumeIfNextToken(final String token) throws InvalidSyntaxException { if (token.equals(getNextToken().token)) return true; @@ -198,6 +170,22 @@ public class Tokenizer { currentIndex = tokenIndexes.pop(); } + /** + * For debugging + */ + public void enlistRemainingTokens(){ + int redTokenCount = 0; + + while (hasMoreContent()) { + out.println(getNextToken().toString()); + redTokenCount++; + } + + // restore pointer to original location + for (int i = 0; i< redTokenCount; i++ ) unreadToken(); + } + + public void skipUntilDataEnd() { tokenIndexes.push(currentIndex); currentIndex = source.length();