import eu.svjatoslav.commons.string.tokenizer.TokenizerMatch;
import eu.svjatoslav.sixth.core.document.text.FormattedText;
-import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
import static eu.svjatoslav.sixth.core.document.Helper.*;
import static eu.svjatoslav.sixth.core.document.text.FormattedText.fromOrg;
// Org list. Examples:
// " + my list title"
// " - my list title"
- tokenizer.addTerminator(PRESERVE, "([ \\t]*)(\\+|-)[ \\t]?(.*)?\\r?\\n", TG_LIST);
+ tokenizer.addTerminator(PRESERVE, "([ \\t]*)(\\+|-)[ \\t]+(.*)?\\r?\\n", TG_LIST);
// " * my list title"
- tokenizer.addTerminator(PRESERVE, "([ \\t]+)(\\*)[ \\t]?(.*)?\\r?\\n", TG_LIST);
+ tokenizer.addTerminator(PRESERVE, "([ \\t]+)(\\*)[ \\t]+(.*)?\\r?\\n", TG_LIST);
+
+ // TODO: add numbered list
// DocumentProperty:
// "#+OPTIONS: H:20 num:20"
- tokenizer.addTerminator(PRESERVE, "#\\+.+:.*\\r?\\n", TG_DOCUMENT_PROPERTY);
+ tokenizer.addTerminator(PRESERVE, "#\\+([^\\s]+):(.*)\\r?\\n", TG_DOCUMENT_PROPERTY);
+
+ // Drawer property:
+ // " :ID: 533734b9-0456-4448-9830-a43646345615"
+ tokenizer.addTerminator(PRESERVE, "([ \\t]*):([^\\s]+):(.*)\\r?\\n", TG_DRAWER_PROPERTY);
+
+
+ // multiline code block
+ tokenizer.addTerminator(PRESERVE,
+ "([ \\t]*)#\\+BEGIN_SRC" + // source begin identifier
+ "(([ \\t]+)(.*))?(\\r?\\n)" + // source block parameters
+ "((?:.|\\n|\\r)*?)" + // source content
+ "(\\r?\\n)([ \\t]*)#\\+END_SRC(.*)\\r?\\n" // source end identifier
+ , TG_MULTILINE_CODE);
+
+ // verse
+ tokenizer.addTerminator(PRESERVE,
+ "([ \\t]*)#\\+BEGIN_VERSE" + // verse begin identifier
+ "(([ \\t]+)(.*))?(\\r?\\n)" + // verse block parameters
+ "((?:.|\\n|\\r)*?)" + // verse
+ "(\\r?\\n)([ \\t]*)#\\+END_VERSE(.*)\\r?\\n" // verse end identifier
+ , TG_VERSE);
+
+
+ // TODO: add support for export blocks:
+ // #+begin_export latex
+ // \clearpage
+ //#+end_export
- // newline
- tokenizer.addTerminator(DROP,"\\r?\\n", TG_NEWLINE);
+ // normal text
+ tokenizer.addTerminator(PRESERVE,".*\\r?\\n", TG_NORMAL_TEXT);
while (tokenizer.hasMoreContent()) {
final TokenizerMatch tm = tokenizer.getNextToken();