939ede9dc5a8a7ecfe687745261254ba3b9b235b
[svjatoslav_commons.git] / src / main / java / eu / svjatoslav / commons / string / tokenizer / Tokenizer.java
1 /*
2  * Svjatoslav Commons - shared library of common functionality.
3  * Copyright ©2012-2017, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu
4  * 
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of version 3 of the GNU Lesser General Public License
7  * or later as published by the Free Software Foundation.
8  */
9
10 package eu.svjatoslav.commons.string.tokenizer;
11
12 import java.util.ArrayList;
13 import java.util.List;
14 import java.util.Stack;
15 import java.util.stream.Stream;
16
17 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
18 import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
19
20 public class Tokenizer {
21
22     final Stack<Integer> tokenIndexes = new Stack<>();
23     private final List<Terminator> terminators = new ArrayList<>();
24     private String source;
25     private int currentIndex = 0;
26
27     public Tokenizer(final String source) {
28         this.source = source;
29     }
30
31     public Tokenizer(){}
32
33     public Tokenizer setSource(String source){
34         this.source = source;
35         currentIndex = 0;
36         tokenIndexes.clear();
37         return this;
38     }
39
40     public Tokenizer addTerminator(final String startSequence,
41                                    final Terminator.TerminationStrategy terminationStrategy) {
42         terminators.add(new Terminator(startSequence, terminationStrategy));
43         return this;
44     }
45
46     public Tokenizer addTerminator(final String startSequence,
47                                    final String endSequence, final Terminator.TerminationStrategy terminationStrategy) {
48         terminators.add(new Terminator(startSequence, endSequence, terminationStrategy));
49         return this;
50     }
51
52     public void expectAndConsumeNextToken(final String value)
53             throws InvalidSyntaxException {
54         final TokenizerMatch match = getNextToken();
55         if (!value.equals(match.token))
56             throw new InvalidSyntaxException("Expected \"" + value
57                     + "\" but got \"" + match.token + "\" instead.");
58     }
59
60     public TokenizerMatch getNextToken() throws InvalidSyntaxException {
61         tokenIndexes.push(currentIndex);
62
63         StringBuilder token = new StringBuilder();
64
65         while (true){
66             if (isOngoingToken()) {
67                 token.append(source.charAt(currentIndex));
68                 currentIndex++;
69                 continue;
70             }
71
72             Terminator tokenTerminator = findTokenTerminator();
73
74             if (tokenTerminator.termination == PRESERVE){
75                 return buildPreservedToken(token, tokenTerminator);
76             } else if (tokenTerminator.termination == DROP){
77                 if (hasAccumulatedToken(token)){
78                     currentIndex++;
79                     return new TokenizerMatch(token.toString(), "", tokenTerminator);
80                 } else {
81                     currentIndex++;
82                 }
83             }
84         }
85
86     }
87
88     private TokenizerMatch buildPreservedToken(StringBuilder token, Terminator terminator) throws InvalidSyntaxException {
89         if (hasAccumulatedToken(token))
90             return new TokenizerMatch(token.toString(), "", terminator);
91
92         if (terminator.hasEndSequence()){
93             int endSequenceIndex = source.indexOf(terminator.endSequence,
94                     currentIndex + terminator.startSequence.length());
95
96             if (endSequenceIndex < 0)
97                 throw new InvalidSyntaxException("Expected \"" + terminator.endSequence + "\" but not found.");
98
99             String reminder = source.substring(currentIndex + terminator.startSequence.length(), endSequenceIndex);
100             currentIndex = endSequenceIndex + terminator.endSequence.length();
101
102             return new TokenizerMatch(terminator.startSequence, reminder, terminator);
103         } else {
104             currentIndex += terminator.startSequence.length();
105             return new TokenizerMatch(terminator.startSequence, "", terminator);
106         }
107     }
108
109     private boolean hasAccumulatedToken(StringBuilder token) {
110         return token.length() > 0;
111     }
112
113     private boolean isOngoingToken() {
114         return findTokenTerminator() == null;
115     }
116
117     public Terminator findTokenTerminator() {
118         for (Terminator terminator : terminators)
119             if (terminator.matches(source, currentIndex))
120                 return terminator;
121         return null;
122     }
123
124     public boolean consumeIfNextToken(final String token) throws InvalidSyntaxException {
125         if (token.equals(getNextToken().token))
126             return true;
127
128         unreadToken();
129         return false;
130     }
131
132     public TokenizerMatch peekNextToken() throws InvalidSyntaxException {
133         TokenizerMatch result = getNextToken();
134         unreadToken();
135         return result;
136     }
137
138     public boolean peekIsOneOf(String ... possibilities) throws InvalidSyntaxException {
139         String nextToken = peekNextToken().token;
140         return Stream.of(possibilities).anyMatch(possibility -> possibility.equals(nextToken));
141     }
142
143     public void peekExpectNoneOf(String ... possibilities) throws InvalidSyntaxException {
144         if (peekIsOneOf(possibilities))
145             throw new InvalidSyntaxException("Not expected \"" + peekNextToken().token + "\" here.");
146     }
147
148
149     public boolean sequenceMatches(final String sequence) {
150         if ((currentIndex + sequence.length()) > source.length())
151             return false;
152
153         for (int i = 0; i < sequence.length(); i++)
154             if (sequence.charAt(i) != source.charAt(i + currentIndex))
155                 return false;
156
157         return true;
158     }
159
160     public void skipUntilDataEnd() {
161         tokenIndexes.push(currentIndex);
162         currentIndex = source.length();
163     }
164
165     public void skipUntilSequence(final String sequence) {
166         while (currentIndex < source.length()) {
167             if (sequenceMatches(sequence)) {
168                 currentIndex += sequence.length();
169                 return;
170             }
171
172             currentIndex++;
173         }
174     }
175
176     public void unreadToken() {
177         currentIndex = tokenIndexes.pop();
178     }
179
180 }