From 05934ede2e526bc55d3661d6029891bef70fd3b9 Mon Sep 17 00:00:00 2001 From: Svjatoslav Agejenko Date: Fri, 12 Jul 2013 00:48:10 +0300 Subject: [PATCH] recognize java comments --- .classpath | 8 +-- .../java/methods/ClassReference.java | 8 +-- .../inspector/java/methods/Clazz.java | 21 ++++-- .../inspector/java/methods/JavaFile.java | 43 ++++++++--- .../inspector/java/methods/Modifiers.java | 7 ++ .../inspector/tokenizer/Terminator.java | 20 ++++-- .../inspector/tokenizer/Tokenizer.java | 71 ++++++++++++------- 7 files changed, 126 insertions(+), 52 deletions(-) diff --git a/.classpath b/.classpath index fd7ad7f..5adac7b 100644 --- a/.classpath +++ b/.classpath @@ -6,19 +6,19 @@ - + - - + - + + diff --git a/src/main/java/eu/svjatoslav/inspector/java/methods/ClassReference.java b/src/main/java/eu/svjatoslav/inspector/java/methods/ClassReference.java index 43d7625..1a943f5 100644 --- a/src/main/java/eu/svjatoslav/inspector/java/methods/ClassReference.java +++ b/src/main/java/eu/svjatoslav/inspector/java/methods/ClassReference.java @@ -14,20 +14,20 @@ public class ClassReference { public ClassReference(final Tokenizer tokenizer) throws InvalidSyntaxException { - name = tokenizer.getToken().token; + name = tokenizer.getNextToken().token; - if (!tokenizer.isNextToken("<")) + if (!tokenizer.probeNextToken("<")) return; while (true) { final ClassReference parameterType = new ClassReference(tokenizer); typeParameters.add(parameterType); - if (!tokenizer.isNextToken(",")) + if (!tokenizer.probeNextToken(",")) break; } - tokenizer.expectToken(">"); + tokenizer.expectNextToken(">"); } @Override diff --git a/src/main/java/eu/svjatoslav/inspector/java/methods/Clazz.java b/src/main/java/eu/svjatoslav/inspector/java/methods/Clazz.java index fda73d1..3c5333b 100644 --- a/src/main/java/eu/svjatoslav/inspector/java/methods/Clazz.java +++ b/src/main/java/eu/svjatoslav/inspector/java/methods/Clazz.java @@ -11,17 +11,21 @@ public class Clazz { private final String packageName; private final String className; + private final boolean isInterface; public ClassReference superClass; public List implementedInterfaces = new ArrayList(); public Clazz(final String packageName, final String className, - final Tokenizer tokenizer) throws InvalidSyntaxException { + final Tokenizer tokenizer, final boolean isInterface) + throws InvalidSyntaxException { + this.packageName = packageName; this.className = className; + this.isInterface = isInterface; while (true) { - final TokenizerMatch match = tokenizer.getToken(); + final TokenizerMatch match = tokenizer.getNextToken(); if ("extends".equals(match.token)) { superClass = new ClassReference(tokenizer); @@ -32,7 +36,7 @@ public class Clazz { while (true) { implementedInterfaces.add(new ClassReference(tokenizer)); - if (tokenizer.isNextToken(",")) + if (tokenizer.probeNextToken(",")) continue; break; @@ -49,14 +53,21 @@ public class Clazz { } public void parseClassBody(final Tokenizer tokenizer) { - tokenizer.skipUtilEnd(); + tokenizer.skipUntilDataEnd(); } @Override public String toString() { final EnumerationBuffer result = new EnumerationBuffer(); - result.append(packageName + " -> " + className + "\n"); + result.append(packageName + " -> " + className + " "); + + if (isInterface) + result.append("(interface)"); + else + result.append("(class)"); + result.append("\n"); + if (superClass != null) result.append(" super: " + superClass.toString() + "\n"); diff --git a/src/main/java/eu/svjatoslav/inspector/java/methods/JavaFile.java b/src/main/java/eu/svjatoslav/inspector/java/methods/JavaFile.java index 400ff87..ea05f01 100644 --- a/src/main/java/eu/svjatoslav/inspector/java/methods/JavaFile.java +++ b/src/main/java/eu/svjatoslav/inspector/java/methods/JavaFile.java @@ -33,6 +33,8 @@ public class JavaFile { readFile(); final Tokenizer tokenizer = new Tokenizer(contents.toString()); + + // empty space tokenizer.addTerminator(" ", true); tokenizer.addTerminator("\t", true); tokenizer.addTerminator("\n", true); @@ -48,10 +50,14 @@ public class JavaFile { tokenizer.addTerminator(">", false); tokenizer.addTerminator(",", false); + // comments + tokenizer.addTerminator("//", "\n", true); + tokenizer.addTerminator("/*", "*/", true); + final Modifiers modifiers = new Modifiers(); while (true) { - final TokenizerMatch match = tokenizer.getToken(); + final TokenizerMatch match = tokenizer.getNextToken(); if (match == null) break; @@ -74,6 +80,11 @@ public class JavaFile { continue; } + if ("interface".equals(match.token)) { + parseInterface(tokenizer); + continue; + } + System.out.println(" " + modifiers.toString() + " " + match.token); modifiers.reset(); @@ -85,10 +96,10 @@ public class JavaFile { private void parseClass(final Tokenizer tokenizer) throws InvalidSyntaxException { - final TokenizerMatch match = tokenizer.getToken(); - final Clazz clazz = new Clazz(packageName, match.token, tokenizer); + final TokenizerMatch match = tokenizer.getNextToken(); + final Clazz clazz = new Clazz(packageName, match.token, tokenizer, + false); System.out.println(clazz.toString()); - } private void parseImport(final Tokenizer tokenizer) @@ -96,27 +107,35 @@ public class JavaFile { final Import imp = new Import(); - final TokenizerMatch match = tokenizer.getToken(); + final TokenizerMatch match = tokenizer.getNextToken(); if (match.token.equals("static")) { imp.isStatic = true; - imp.path = tokenizer.getToken().token; + imp.path = tokenizer.getNextToken().token; } else imp.path = match.token; imports.add(imp); - tokenizer.expectToken(";"); + tokenizer.expectNextToken(";"); + } + + private void parseInterface(final Tokenizer tokenizer) + throws InvalidSyntaxException { + + final TokenizerMatch match = tokenizer.getNextToken(); + final Clazz clazz = new Clazz(packageName, match.token, tokenizer, true); + System.out.println(clazz.toString()); } private void parsePackage(final Tokenizer tokenizer) throws InvalidSyntaxException { - final TokenizerMatch match = tokenizer.getToken(); + final TokenizerMatch match = tokenizer.getNextToken(); packageName = match.token; - tokenizer.expectToken(";"); + tokenizer.expectNextToken(";"); } private void readFile() throws FileNotFoundException, IOException { @@ -140,7 +159,11 @@ public class JavaFile { public void skipUntilSemicolon(final Tokenizer tokenizer) { while (true) { - final TokenizerMatch token = tokenizer.getToken(); + final TokenizerMatch token = tokenizer.getNextToken(); + + if (token == null) + return; + if (token.token.equals(";")) return; } diff --git a/src/main/java/eu/svjatoslav/inspector/java/methods/Modifiers.java b/src/main/java/eu/svjatoslav/inspector/java/methods/Modifiers.java index 747d7db..947b3f4 100644 --- a/src/main/java/eu/svjatoslav/inspector/java/methods/Modifiers.java +++ b/src/main/java/eu/svjatoslav/inspector/java/methods/Modifiers.java @@ -19,6 +19,8 @@ public class Modifiers { boolean isFinal = false; + boolean isAbstract = false; + public boolean parseModifier(final String string) { for (final Access access : Access.values()) if (access.name.equals(string)) { @@ -36,6 +38,11 @@ public class Modifiers { return true; } + if ("abstract".equals(string)) { + isAbstract = true; + return true; + } + return false; } diff --git a/src/main/java/eu/svjatoslav/inspector/tokenizer/Terminator.java b/src/main/java/eu/svjatoslav/inspector/tokenizer/Terminator.java index ae49c41..8848775 100644 --- a/src/main/java/eu/svjatoslav/inspector/tokenizer/Terminator.java +++ b/src/main/java/eu/svjatoslav/inspector/tokenizer/Terminator.java @@ -2,11 +2,21 @@ package eu.svjatoslav.inspector.tokenizer; public class Terminator { - String value; - boolean empty; + String startSequence; + String endSequence; + boolean ignoreTerminator; - public Terminator(final String value, final boolean empty) { - this.value = value; - this.empty = empty; + public Terminator(final String startPattern, final boolean ignoreTerminator) { + this.startSequence = startPattern; + this.ignoreTerminator = ignoreTerminator; } + + public Terminator(final String startSequence, final String endSequence, + final boolean ignoreTerminator) { + + this.startSequence = startSequence; + this.endSequence = endSequence; + this.ignoreTerminator = ignoreTerminator; + } + } diff --git a/src/main/java/eu/svjatoslav/inspector/tokenizer/Tokenizer.java b/src/main/java/eu/svjatoslav/inspector/tokenizer/Tokenizer.java index 8a42776..ec13b1a 100644 --- a/src/main/java/eu/svjatoslav/inspector/tokenizer/Tokenizer.java +++ b/src/main/java/eu/svjatoslav/inspector/tokenizer/Tokenizer.java @@ -17,19 +17,26 @@ public class Tokenizer { this.source = source; } - public void addTerminator(final String terminator, final boolean empty) { + public void addTerminator(final String startSequence, + final boolean ignoreTerminator) { + terminators.add(new Terminator(startSequence, ignoreTerminator)); + } - terminators.add(new Terminator(terminator, empty)); + public void addTerminator(final String startSequence, + final String endSequence, final boolean ignoreTerminator) { + terminators.add(new Terminator(startSequence, endSequence, + ignoreTerminator)); } - public void expectToken(final String value) throws InvalidSyntaxException { - final TokenizerMatch match = getToken(); + public void expectNextToken(final String value) + throws InvalidSyntaxException { + final TokenizerMatch match = getNextToken(); if (!value.equals(match.token)) throw new InvalidSyntaxException("Expected \"" + value + "\" but got \"" + match.token + "\" instead."); } - public TokenizerMatch getToken() { + public TokenizerMatch getNextToken() { tokenIndexes.push(currentIndex); final StringBuffer result = new StringBuffer(); @@ -40,10 +47,14 @@ public class Tokenizer { boolean accumulateCurrentChar = true; findTerminator: for (final Terminator terminator : terminators) - if (terminatorMatches(terminator)) - // empty space detected - if (terminator.empty) { - currentIndex += terminator.value.length(); + if (sequenceMatches(terminator.startSequence)) + + if (terminator.ignoreTerminator) { + currentIndex += terminator.startSequence.length(); + + if (terminator.endSequence != null) + skipUntilSequence(terminator.endSequence); + if (result.length() > 0) return new TokenizerMatch(result.toString(), terminator); @@ -54,8 +65,9 @@ public class Tokenizer { } else if (result.length() > 0) return new TokenizerMatch(result.toString(), terminator); else { - currentIndex += terminator.value.length(); - return new TokenizerMatch(terminator.value, terminator); + currentIndex += terminator.startSequence.length(); + return new TokenizerMatch(terminator.startSequence, + terminator); } if (accumulateCurrentChar) { @@ -66,32 +78,43 @@ public class Tokenizer { } - public boolean isNextToken(final String token) { - if (token.equals(getToken().token)) + public boolean probeNextToken(final String token) { + if (token.equals(getNextToken().token)) return true; - rollbackToken(); + unreadToken(); return false; } - public void rollbackToken() { - currentIndex = tokenIndexes.pop(); + public boolean sequenceMatches(final String sequence) { + if ((currentIndex + sequence.length()) > source.length()) + return false; + + for (int i = 0; i < sequence.length(); i++) + if (sequence.charAt(i) != source.charAt(i + currentIndex)) + return false; + + return true; } - public void skipUtilEnd() { + public void skipUntilDataEnd() { tokenIndexes.push(currentIndex); currentIndex = source.length(); } - public boolean terminatorMatches(final Terminator terminator) { - if ((currentIndex + terminator.value.length()) > source.length()) - return false; + public void skipUntilSequence(final String sequence) { + while (currentIndex < source.length()) { + if (sequenceMatches(sequence)) { + currentIndex += sequence.length(); + return; + } - for (int i = 0; i < terminator.value.length(); i++) - if (terminator.value.charAt(i) != source.charAt(i + currentIndex)) - return false; + currentIndex++; + } + } - return true; + public void unreadToken() { + currentIndex = tokenIndexes.pop(); } } -- 2.20.1