import eu.svjatoslav.commons.string.tokenizer.TokenizerMatch;
import eu.svjatoslav.sixth.core.document.text.FormattedText;
-import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.DROP;
import static eu.svjatoslav.commons.string.tokenizer.Terminator.TerminationStrategy.PRESERVE;
import static eu.svjatoslav.sixth.core.document.Helper.*;
import static eu.svjatoslav.sixth.core.document.text.FormattedText.fromOrg;
// Org list. Examples:
// " + my list title"
// " - my list title"
- tokenizer.addTerminator(PRESERVE, "([ \\t]*)(\\+|-)[ \\t]?(.*)?\\r?\\n", TG_LIST);
+ tokenizer.addTerminator(PRESERVE, "([ \\t]*)(\\+|-)[ \\t]+(.*)?\\r?\\n", TG_LIST);
// " * my list title"
- tokenizer.addTerminator(PRESERVE, "([ \\t]+)(\\*)[ \\t]?(.*)?\\r?\\n", TG_LIST);
+ tokenizer.addTerminator(PRESERVE, "([ \\t]+)(\\*)[ \\t]+(.*)?\\r?\\n", TG_LIST);
// DocumentProperty:
// "#+OPTIONS: H:20 num:20"
tokenizer.addTerminator(PRESERVE, "#\\+.+:.*\\r?\\n", TG_DOCUMENT_PROPERTY);
// newline
- tokenizer.addTerminator(DROP,"\\r?\\n", TG_NEWLINE);
+ tokenizer.addTerminator(PRESERVE,".*\\r?\\n", TG_NORMAL_TEXT);
while (tokenizer.hasMoreContent()) {