1 #include <iostream> 2 #include <map> 3 #include <memory> 4 #include <sstream> 5 6 #include "../src/check.h" // NOTE: check.h is for internal use only! 7 #include "../src/re.h" // NOTE: re.h is for internal use only 8 #include "output_test.h" 9 10 // ========================================================================= // 11 // ------------------------------ Internals -------------------------------- // 12 // ========================================================================= // 13 namespace internal { 14 namespace { 15 16 using TestCaseList = std::vector<TestCase>; 17 18 // Use a vector because the order elements are added matters during iteration. 19 // std::map/unordered_map don't guarantee that. 20 // For example: 21 // SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}}); 22 // Substitute("%HelloWorld") // Always expands to Hello. 23 using SubMap = std::vector<std::pair<std::string, std::string>>; 24 25 TestCaseList& GetTestCaseList(TestCaseID ID) { 26 // Uses function-local statics to ensure initialization occurs 27 // before first use. 28 static TestCaseList lists[TC_NumID]; 29 return lists[ID]; 30 } 31 32 SubMap& GetSubstitutions() { 33 // Don't use 'dec_re' from header because it may not yet be initialized. 34 static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; 35 static SubMap map = { 36 {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, 37 {"%int", "[ ]*[0-9]+"}, 38 {" %s ", "[ ]+"}, 39 {"%time", "[ ]*[0-9]{1,5} ns"}, 40 {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"}, 41 {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"}, 42 {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"}, 43 {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, 44 {"%csv_bytes_report", 45 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, 46 {"%csv_items_report", 47 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"}, 48 {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"}, 49 {"%csv_label_report_end", ",,"}}; 50 return map; 51 } 52 53 std::string PerformSubstitutions(std::string source) { 54 SubMap const& subs = GetSubstitutions(); 55 using SizeT = std::string::size_type; 56 for (auto const& KV : subs) { 57 SizeT pos; 58 SizeT next_start = 0; 59 while ((pos = source.find(KV.first, next_start)) != std::string::npos) { 60 next_start = pos + KV.second.size(); 61 source.replace(pos, KV.first.size(), KV.second); 62 } 63 } 64 return source; 65 } 66 67 void CheckCase(std::stringstream& remaining_output, TestCase const& TC, 68 TestCaseList const& not_checks) { 69 std::string first_line; 70 bool on_first = true; 71 std::string line; 72 while (remaining_output.eof() == false) { 73 CHECK(remaining_output.good()); 74 std::getline(remaining_output, line); 75 if (on_first) { 76 first_line = line; 77 on_first = false; 78 } 79 for (const auto& NC : not_checks) { 80 CHECK(!NC.regex->Match(line)) 81 << "Unexpected match for line \"" << line << "\" for MR_Not regex \"" 82 << NC.regex_str << "\"" 83 << "\n actual regex string \"" << TC.substituted_regex << "\"" 84 << "\n started matching near: " << first_line; 85 } 86 if (TC.regex->Match(line)) return; 87 CHECK(TC.match_rule != MR_Next) 88 << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str 89 << "\"" 90 << "\n actual regex string \"" << TC.substituted_regex << "\"" 91 << "\n started matching near: " << first_line; 92 } 93 CHECK(remaining_output.eof() == false) 94 << "End of output reached before match for regex \"" << TC.regex_str 95 << "\" was found" 96 << "\n actual regex string \"" << TC.substituted_regex << "\"" 97 << "\n started matching near: " << first_line; 98 } 99 100 void CheckCases(TestCaseList const& checks, std::stringstream& output) { 101 std::vector<TestCase> not_checks; 102 for (size_t i = 0; i < checks.size(); ++i) { 103 const auto& TC = checks[i]; 104 if (TC.match_rule == MR_Not) { 105 not_checks.push_back(TC); 106 continue; 107 } 108 CheckCase(output, TC, not_checks); 109 not_checks.clear(); 110 } 111 } 112 113 class TestReporter : public benchmark::BenchmarkReporter { 114 public: 115 TestReporter(std::vector<benchmark::BenchmarkReporter*> reps) 116 : reporters_(reps) {} 117 118 virtual bool ReportContext(const Context& context) { 119 bool last_ret = false; 120 bool first = true; 121 for (auto rep : reporters_) { 122 bool new_ret = rep->ReportContext(context); 123 CHECK(first || new_ret == last_ret) 124 << "Reports return different values for ReportContext"; 125 first = false; 126 last_ret = new_ret; 127 } 128 (void)first; 129 return last_ret; 130 } 131 132 void ReportRuns(const std::vector<Run>& report) { 133 for (auto rep : reporters_) rep->ReportRuns(report); 134 } 135 void Finalize() { 136 for (auto rep : reporters_) rep->Finalize(); 137 } 138 139 private: 140 std::vector<benchmark::BenchmarkReporter *> reporters_; 141 }; 142 } 143 } // end namespace internal 144 145 // ========================================================================= // 146 // -------------------------- Public API Definitions------------------------ // 147 // ========================================================================= // 148 149 TestCase::TestCase(std::string re, int rule) 150 : regex_str(std::move(re)), 151 match_rule(rule), 152 substituted_regex(internal::PerformSubstitutions(regex_str)), 153 regex(std::make_shared<benchmark::Regex>()) { 154 std::string err_str; 155 regex->Init(substituted_regex,& err_str); 156 CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex 157 << "\"" 158 << "\n originally \"" << regex_str << "\"" 159 << "\n got error: " << err_str; 160 } 161 162 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) { 163 auto& L = internal::GetTestCaseList(ID); 164 L.insert(L.end(), il); 165 return 0; 166 } 167 168 int SetSubstitutions( 169 std::initializer_list<std::pair<std::string, std::string>> il) { 170 auto& subs = internal::GetSubstitutions(); 171 for (auto KV : il) { 172 bool exists = false; 173 KV.second = internal::PerformSubstitutions(KV.second); 174 for (auto& EKV : subs) { 175 if (EKV.first == KV.first) { 176 EKV.second = std::move(KV.second); 177 exists = true; 178 break; 179 } 180 } 181 if (!exists) subs.push_back(std::move(KV)); 182 } 183 return 0; 184 } 185 186 void RunOutputTests(int argc, char* argv[]) { 187 using internal::GetTestCaseList; 188 benchmark::Initialize(&argc, argv); 189 benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None); 190 benchmark::JSONReporter JR; 191 benchmark::CSVReporter CSVR; 192 struct ReporterTest { 193 const char* name; 194 std::vector<TestCase>& output_cases; 195 std::vector<TestCase>& error_cases; 196 benchmark::BenchmarkReporter& reporter; 197 std::stringstream out_stream; 198 std::stringstream err_stream; 199 200 ReporterTest(const char* n, std::vector<TestCase>& out_tc, 201 std::vector<TestCase>& err_tc, 202 benchmark::BenchmarkReporter& br) 203 : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { 204 reporter.SetOutputStream(&out_stream); 205 reporter.SetErrorStream(&err_stream); 206 } 207 } TestCases[] = { 208 {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), 209 GetTestCaseList(TC_ConsoleErr), CR}, 210 {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), 211 JR}, 212 {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), 213 CSVR}, 214 }; 215 216 // Create the test reporter and run the benchmarks. 217 std::cout << "Running benchmarks...\n"; 218 internal::TestReporter test_rep({&CR, &JR, &CSVR}); 219 benchmark::RunSpecifiedBenchmarks(&test_rep); 220 221 for (auto& rep_test : TestCases) { 222 std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; 223 std::string banner(msg.size() - 1, '-'); 224 std::cout << banner << msg << banner << "\n"; 225 226 std::cerr << rep_test.err_stream.str(); 227 std::cout << rep_test.out_stream.str(); 228 229 internal::CheckCases(rep_test.error_cases, rep_test.err_stream); 230 internal::CheckCases(rep_test.output_cases, rep_test.out_stream); 231 232 std::cout << "\n"; 233 } 234 } 235
