+ final Tokenizer tokenizer = new Tokenizer(contents.toString());
+
+ // empty space
+ tokenizer.addTerminator(" ", DROP);
+ tokenizer.addTerminator("\t", DROP);
+ tokenizer.addTerminator("\n", DROP);
+
+ tokenizer.addTerminator(";", PRESERVE);
+ tokenizer.addTerminator("{", PRESERVE);
+ tokenizer.addTerminator("}", PRESERVE);
+ tokenizer.addTerminator("(", PRESERVE);
+ tokenizer.addTerminator(")", PRESERVE);
+ tokenizer.addTerminator("[", PRESERVE);
+ tokenizer.addTerminator("]", PRESERVE);
+ tokenizer.addTerminator("<", PRESERVE);
+ tokenizer.addTerminator(">", PRESERVE);
+ tokenizer.addTerminator(",", PRESERVE);
+ tokenizer.addTerminator("@", PRESERVE);