// " * my list title"
tokenizer.addTerminator(PRESERVE, "([ \\t]+)(\\*)[ \\t]+(.*)?\\r?\\n", TG_LIST);
+ // TODO: add numbered list
+
// DocumentProperty:
// "#+OPTIONS: H:20 num:20"
- tokenizer.addTerminator(PRESERVE, "#\\+.+:.*\\r?\\n", TG_DOCUMENT_PROPERTY);
+ tokenizer.addTerminator(PRESERVE, "#\\+[^\\s]+:.*\\r?\\n", TG_DOCUMENT_PROPERTY);
- // newline
- tokenizer.addTerminator(PRESERVE,".*\\r?\\n", TG_NORMAL_TEXT);
+ // Drawer property:
+ // " :ID: 533734b9-0456-4448-9830-a43646345615"
+ tokenizer.addTerminator(PRESERVE, "([ \\t]*):([^\\s]+):(.*)\\r?\\n", TG_DRAWER_PROPERTY);
+
+
+ // multiline code block
+ tokenizer.addTerminator(PRESERVE,
+ "([ \\t]*)#\\+BEGIN_SRC" + // source begin identifier
+ "(([ \\t]+)(.*))?(\\r?\\n)" + // source block parameters
+ "((?:.|\\n|\\r)*?)" + // source content
+ "(\\r?\\n)([ \\t]*)#\\+END_SRC(.*)\\r?\\n" // source end identifier
+ , TG_MULTILINE_CODE);
+
+ // verse
+ tokenizer.addTerminator(PRESERVE,
+ "([ \\t]*)#\\+BEGIN_VERSE" + // verse begin identifier
+ "(([ \\t]+)(.*))?(\\r?\\n)" + // verse block parameters
+ "((?:.|\\n|\\r)*?)" + // verse
+ "(\\r?\\n)([ \\t]*)#\\+END_VERSE(.*)\\r?\\n" // verse end identifier
+ , TG_VERSE);
+ // TODO: add support for export blocks:
+ // #+begin_export latex
+ // \clearpage
+ //#+end_export
+
+
+ // normal text
+ tokenizer.addTerminator(PRESERVE,".*\\r?\\n", TG_NORMAL_TEXT);
+
while (tokenizer.hasMoreContent()) {
final TokenizerMatch tm = tokenizer.getNextToken();