Lines Matching refs:tokenizer
31 #include "Tokenizer.h"
44 SCENARIO("Tokenizer tests")
46 GIVEN ("A default tokenizer") {
49 Tokenizer tokenizer("a bcd ef");
53 CHECK(tokenizer.split() == expected);
58 Tokenizer tokenizer("");
62 CHECK(tokenizer.split() == expected);
67 Tokenizer tokenizer(" a \n\t bc ");
71 CHECK(tokenizer.split() == expected);
76 GIVEN ("A slash-separated string and tokenizer") {
77 Tokenizer tokenizer("/a/bcd/ef g/h/", "/");
81 CHECK(tokenizer.split() == expected);
85 GIVEN ("A tokenizer that doesn't merge consecutive separators") {
88 Tokenizer tokenizer("", Tokenizer::defaultDelimiters, false);
92 CHECK(tokenizer.split() == expected);
97 Tokenizer tokenizer(",", ",", false);
101 CHECK(tokenizer.split() == expected);
106 Tokenizer tokenizer(" a b \nc d ", Tokenizer::defaultDelimiters, false);
110 CHECK(tokenizer.split() == expected);
