X-Git-Url: http://www2.svjatoslav.eu/gitweb/?p=svjatoslav_commons.git;a=blobdiff_plain;f=src%2Fmain%2Fjava%2Feu%2Fsvjatoslav%2Fcommons%2Fstring%2Ftokenizer%2FTokenizer.java;h=1905d9a14220c96783efcd1e2e7dfe589fce8127;hp=b6c0eee3c0d2eb98dbcfd169615a39ae6237da8a;hb=9bf004ce4e9b5edff36c65fcc8cc0f303390d7fc;hpb=afaa928dd10304ee3e8e6bad3a377ced6a7b2f42

diff --git a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
index b6c0eee..1905d9a 100755
--- a/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
+++ b/src/main/java/eu/svjatoslav/commons/string/tokenizer/Tokenizer.java
@@ -15,115 +15,113 @@ import java.util.Stack;
 
 public class Tokenizer {
 
-	private final List<Terminator> terminators = new ArrayList<Terminator>();
-	private final String source;
-
-	Stack<Integer> tokenIndexes = new Stack<Integer>();
-
-	private int currentIndex = 0;
-
-	public Tokenizer(final String source) {
-		this.source = source;
-	}
-
-	public void addTerminator(final String startSequence,
-			final boolean ignoreTerminator) {
-		terminators.add(new Terminator(startSequence, ignoreTerminator));
-	}
-
-	public void addTerminator(final String startSequence,
-			final String endSequence, final boolean ignoreTerminator) {
-		terminators.add(new Terminator(startSequence, endSequence,
-				ignoreTerminator));
-	}
-
-	public void expectNextToken(final String value)
-			throws InvalidSyntaxException {
-		final TokenizerMatch match = getNextToken();
-		if (!value.equals(match.token))
-			throw new InvalidSyntaxException("Expected \"" + value
-					+ "\" but got \"" + match.token + "\" instead.");
-	}
-
-	public TokenizerMatch getNextToken() {
-		tokenIndexes.push(currentIndex);
-		final StringBuffer result = new StringBuffer();
-
-		while (true) {
-			if (currentIndex >= source.length())
-				return null;
-
-			boolean accumulateCurrentChar = true;
-
-			findTerminator: for (final Terminator terminator : terminators)
-				if (sequenceMatches(terminator.startSequence))
-
-					if (terminator.ignoreTerminator) {
-						currentIndex += terminator.startSequence.length();
-
-						if (terminator.endSequence != null)
-							skipUntilSequence(terminator.endSequence);
-
-						if (result.length() > 0)
-							return new TokenizerMatch(result.toString(),
-									terminator);
-						else {
-							accumulateCurrentChar = false;
-							break findTerminator;
-						}
-					} else if (result.length() > 0)
-						return new TokenizerMatch(result.toString(), terminator);
-					else {
-						currentIndex += terminator.startSequence.length();
-						return new TokenizerMatch(terminator.startSequence,
-								terminator);
-					}
-
-			if (accumulateCurrentChar) {
-				result.append(source.charAt(currentIndex));
-				currentIndex++;
-			}
-		}
-
-	}
-
-	public boolean probeNextToken(final String token) {
-		if (token.equals(getNextToken().token))
-			return true;
-
-		unreadToken();
-		return false;
-	}
-
-	public boolean sequenceMatches(final String sequence) {
-		if ((currentIndex + sequence.length()) > source.length())
-			return false;
-
-		for (int i = 0; i < sequence.length(); i++)
-			if (sequence.charAt(i) != source.charAt(i + currentIndex))
-				return false;
-
-		return true;
-	}
-
-	public void skipUntilDataEnd() {
-		tokenIndexes.push(currentIndex);
-		currentIndex = source.length();
-	}
-
-	public void skipUntilSequence(final String sequence) {
-		while (currentIndex < source.length()) {
-			if (sequenceMatches(sequence)) {
-				currentIndex += sequence.length();
-				return;
-			}
-
-			currentIndex++;
-		}
-	}
-
-	public void unreadToken() {
-		currentIndex = tokenIndexes.pop();
-	}
+    final Stack<Integer> tokenIndexes = new Stack<>();
+    private final List<Terminator> terminators = new ArrayList<>();
+    private final String source;
+    private int currentIndex = 0;
+
+    public Tokenizer(final String source) {
+        this.source = source;
+    }
+
+    public void addTerminator(final String startSequence,
+                              final boolean ignoreTerminator) {
+        terminators.add(new Terminator(startSequence, ignoreTerminator));
+    }
+
+    public void addTerminator(final String startSequence,
+                              final String endSequence, final boolean ignoreTerminator) {
+        terminators.add(new Terminator(startSequence, endSequence,
+                ignoreTerminator));
+    }
+
+    public void expectNextToken(final String value)
+            throws InvalidSyntaxException {
+        final TokenizerMatch match = getNextToken();
+        if (!value.equals(match.token))
+            throw new InvalidSyntaxException("Expected \"" + value
+                    + "\" but got \"" + match.token + "\" instead.");
+    }
+
+    public TokenizerMatch getNextToken() {
+        tokenIndexes.push(currentIndex);
+        final StringBuilder result = new StringBuilder();
+
+        while (true) {
+            if (currentIndex >= source.length())
+                return null;
+
+            boolean accumulateCurrentChar = true;
+
+            for (final Terminator terminator : terminators)
+                if (sequenceMatches(terminator.startSequence))
+
+                    if (terminator.ignoreTerminator) {
+                        currentIndex += terminator.startSequence.length();
+
+                        if (terminator.endSequence != null)
+                            skipUntilSequence(terminator.endSequence);
+
+                        if (result.length() > 0)
+                            return new TokenizerMatch(result.toString(),
+                                    terminator);
+                        else {
+                            accumulateCurrentChar = false;
+                            break;
+                        }
+                    } else if (result.length() > 0)
+                        return new TokenizerMatch(result.toString(), terminator);
+                    else {
+                        currentIndex += terminator.startSequence.length();
+                        return new TokenizerMatch(terminator.startSequence,
+                                terminator);
+                    }
+
+            if (accumulateCurrentChar) {
+                result.append(source.charAt(currentIndex));
+                currentIndex++;
+            }
+        }
+
+    }
+
+    public boolean probeNextToken(final String token) {
+        if (token.equals(getNextToken().token))
+            return true;
+
+        unreadToken();
+        return false;
+    }
+
+    public boolean sequenceMatches(final String sequence) {
+        if ((currentIndex + sequence.length()) > source.length())
+            return false;
+
+        for (int i = 0; i < sequence.length(); i++)
+            if (sequence.charAt(i) != source.charAt(i + currentIndex))
+                return false;
+
+        return true;
+    }
+
+    public void skipUntilDataEnd() {
+        tokenIndexes.push(currentIndex);
+        currentIndex = source.length();
+    }
+
+    public void skipUntilSequence(final String sequence) {
+        while (currentIndex < source.length()) {
+            if (sequenceMatches(sequence)) {
+                currentIndex += sequence.length();
+                return;
+            }
+
+            currentIndex++;
+        }
+    }
+
+    public void unreadToken() {
+        currentIndex = tokenIndexes.pop();
+    }
 
 }