Use regular expression tokenizer. WIP

[sixth.git] / src / main / java / eu / svjatoslav / sixth / core / document / OrgParser.java

diff --git a/src/main/java/eu/svjatoslav/sixth/core/document/OrgParser.java b/src/main/java/eu/svjatoslav/sixth/core/document/OrgParser.java
index fdbd41b..8822d21 100644 (file)
--- a/src/main/java/eu/svjatoslav/sixth/core/document/OrgParser.java
+++ b/src/main/java/eu/svjatoslav/sixth/core/document/OrgParser.java
@@ -1,17 +1,11 @@
 package eu.svjatoslav.sixth.core.document;
 
-import eu.svjatoslav.commons.string.String2;
 import eu.svjatoslav.commons.string.tokenizer.InvalidSyntaxException;
-import eu.svjatoslav.commons.string.tokenizer.Tokenizer;
-import eu.svjatoslav.commons.string.tokenizer.TokenizerMatch;
 
 import java.io.File;
 import java.io.IOException;
 
 import static eu.svjatoslav.commons.file.IOHelper.getFileContentsAsString;
-import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
-import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
-import static eu.svjatoslav.sixth.core.document.text.FormattedText.fromOrg;
 
 public class OrgParser {
 
@@ -20,125 +14,11 @@ public class OrgParser {
     public Document parse(File file) throws IOException, InvalidSyntaxException {
         document = new Document();
 
-        Tokenizer lineTokenizer = getFileToLineTokenizer(getFileContentsAsString(file));
+        String fileContentsAsString = getFileContentsAsString(file);
 
-        while (true) {
-            final TokenizerMatch line = lineTokenizer.getNextToken();
-            if (line == null)
-                break; // EOF
-
-            parseLine(line.token);
-        }
+        document.parse(fileContentsAsString);
 
         return document;
     }
 
-    private void parseLine(String line) throws InvalidSyntaxException {
-        Tokenizer lineTokenizer = getLineTokenizer(line + "\n");
-
-        TokenizerMatch token = lineTokenizer.getNextToken();
-        if (token == null) return;
-
-        if (token.terminator == null)
-            return;
-
-        if (token.token.startsWith("*")){
-            parseHeading(token);
-            return;
-        }
-    }
-
-    private void parseHeading(TokenizerMatch token) {
-        int level = token.token.length()-1;
-        document.createHeading(fromOrg(token.reminder), level);
-   }
-
-    private Tokenizer getLineTokenizer(String contents) {
-        final Tokenizer tokenizer = new Tokenizer(contents);
-        for (int i = 1; i<50; i++){
-            String prefix = new String2("*").repeat(i).toString();
-            tokenizer.addTerminator(prefix +" ","\n", PRESERVE);
-        }
-        return tokenizer;
-    }
-
-    private Tokenizer getFileToLineTokenizer(String contents) {
-        final Tokenizer tokenizer = new Tokenizer(contents);
-
-        // empty space
-//        tokenizer.addTerminator(" ", DROP);
-//        tokenizer.addTerminator("\t", DROP);
-//        tokenizer.addTerminator("\n", DROP);
-
-        // newline
-        tokenizer.addTerminator("\n", DROP);
-
-//        tokenizer.addTerminator(";", PRESERVE);
-//        tokenizer.addTerminator("{", PRESERVE);
-//        tokenizer.addTerminator("}", PRESERVE);
-//        tokenizer.addTerminator("(", PRESERVE);
-//        tokenizer.addTerminator(")", PRESERVE);
-//        tokenizer.addTerminator("[", PRESERVE);
-//        tokenizer.addTerminator("]", PRESERVE);
-//        tokenizer.addTerminator("<", PRESERVE);
-//        tokenizer.addTerminator(">", PRESERVE);
-//        tokenizer.addTerminator(",", PRESERVE);
-//        tokenizer.addTerminator("@", PRESERVE);
-
-        // comments
-//        tokenizer.addTerminator("//", "\n", DROP);
-//        tokenizer.addTerminator("/*", "*/", DROP);
-        return tokenizer;
-    }
-//
-//    private void parseImport(final Tokenizer tokenizer)
-//            throws InvalidSyntaxException {
-//
-//        final Import imp = new Import();
-//
-//        final TokenizerMatch match = tokenizer.getNextToken();
-//
-//        if (match.token.equals("static")) {
-//            imp.isStatic = true;
-//            imp.path = tokenizer.getNextToken().token;
-//        } else
-//            imp.path = match.token;
-//
-//        imports.add(imp);
-//
-//        tokenizer.expectAndConsumeNextToken(";");
-//    }
-//
-//    private void parseInterface(final Tokenizer tokenizer)
-//            throws InvalidSyntaxException {
-//
-//        final TokenizerMatch match = tokenizer.getNextToken();
-//        final Clazz clazz = new Clazz(packageName, match.token, tokenizer, true);
-//        // System.out.println(clazz.toString());
-//        classes.add(clazz);
-//    }
-//
-//    private void parsePackage(final Tokenizer tokenizer)
-//            throws InvalidSyntaxException {
-//
-//        final TokenizerMatch match = tokenizer.getNextToken();
-//
-//        packageName = match.token;
-//
-//        tokenizer.expectAndConsumeNextToken(";");
-//    }
-
-    public void skipUntilSemicolon(final Tokenizer tokenizer) throws InvalidSyntaxException {
-        while (true) {
-            final TokenizerMatch token = tokenizer.getNextToken();
-
-            if (token == null)
-                return;
-
-            if (token.token.equals(";"))
-                return;
-        }
-    }
-
-
 }