1 #include <iostream> 2 #include <map> 3 #include <memory> 4 #include <sstream> 5 #include <cstring> 6 7 #include "../src/check.h" // NOTE: check.h is for internal use only! 8 #include "../src/re.h" // NOTE: re.h is for internal use only 9 #include "output_test.h" 10 #include "../src/benchmark_api_internal.h" 11 12 // ========================================================================= // 13 // ------------------------------ Internals -------------------------------- // 14 // ========================================================================= // 15 namespace internal { 16 namespace { 17 18 using TestCaseList = std::vector<TestCase>; 19 20 // Use a vector because the order elements are added matters during iteration. 21 // std::map/unordered_map don't guarantee that. 22 // For example: 23 // SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}}); 24 // Substitute("%HelloWorld") // Always expands to Hello. 25 using SubMap = std::vector<std::pair<std::string, std::string>>; 26 27 TestCaseList& GetTestCaseList(TestCaseID ID) { 28 // Uses function-local statics to ensure initialization occurs 29 // before first use. 30 static TestCaseList lists[TC_NumID]; 31 return lists[ID]; 32 } 33 34 SubMap& GetSubstitutions() { 35 // Don't use 'dec_re' from header because it may not yet be initialized. 36 static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; 37 static SubMap map = { 38 {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, 39 // human-readable float 40 {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, 41 {"%int", "[ ]*[0-9]+"}, 42 {" %s ", "[ ]+"}, 43 {"%time", "[ ]*[0-9]{1,5} ns"}, 44 {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"}, 45 {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"}, 46 {"%csv_header", 47 "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," 48 "items_per_second,label,error_occurred,error_message"}, 49 {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"}, 50 {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, 51 {"%csv_bytes_report", 52 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, 53 {"%csv_items_report", 54 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"}, 55 {"%csv_bytes_items_report", 56 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + 57 "," + safe_dec_re + ",,,"}, 58 {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"}, 59 {"%csv_label_report_end", ",,"}}; 60 return map; 61 } 62 63 std::string PerformSubstitutions(std::string source) { 64 SubMap const& subs = GetSubstitutions(); 65 using SizeT = std::string::size_type; 66 for (auto const& KV : subs) { 67 SizeT pos; 68 SizeT next_start = 0; 69 while ((pos = source.find(KV.first, next_start)) != std::string::npos) { 70 next_start = pos + KV.second.size(); 71 source.replace(pos, KV.first.size(), KV.second); 72 } 73 } 74 return source; 75 } 76 77 void CheckCase(std::stringstream& remaining_output, TestCase const& TC, 78 TestCaseList const& not_checks) { 79 std::string first_line; 80 bool on_first = true; 81 std::string line; 82 while (remaining_output.eof() == false) { 83 CHECK(remaining_output.good()); 84 std::getline(remaining_output, line); 85 if (on_first) { 86 first_line = line; 87 on_first = false; 88 } 89 for (const auto& NC : not_checks) { 90 CHECK(!NC.regex->Match(line)) 91 << "Unexpected match for line \"" << line << "\" for MR_Not regex \"" 92 << NC.regex_str << "\"" 93 << "\n actual regex string \"" << TC.substituted_regex << "\"" 94 << "\n started matching near: " << first_line; 95 } 96 if (TC.regex->Match(line)) return; 97 CHECK(TC.match_rule != MR_Next) 98 << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str 99 << "\"" 100 << "\n actual regex string \"" << TC.substituted_regex << "\"" 101 << "\n started matching near: " << first_line; 102 } 103 CHECK(remaining_output.eof() == false) 104 << "End of output reached before match for regex \"" << TC.regex_str 105 << "\" was found" 106 << "\n actual regex string \"" << TC.substituted_regex << "\"" 107 << "\n started matching near: " << first_line; 108 } 109 110 void CheckCases(TestCaseList const& checks, std::stringstream& output) { 111 std::vector<TestCase> not_checks; 112 for (size_t i = 0; i < checks.size(); ++i) { 113 const auto& TC = checks[i]; 114 if (TC.match_rule == MR_Not) { 115 not_checks.push_back(TC); 116 continue; 117 } 118 CheckCase(output, TC, not_checks); 119 not_checks.clear(); 120 } 121 } 122 123 class TestReporter : public benchmark::BenchmarkReporter { 124 public: 125 TestReporter(std::vector<benchmark::BenchmarkReporter*> reps) 126 : reporters_(reps) {} 127 128 virtual bool ReportContext(const Context& context) { 129 bool last_ret = false; 130 bool first = true; 131 for (auto rep : reporters_) { 132 bool new_ret = rep->ReportContext(context); 133 CHECK(first || new_ret == last_ret) 134 << "Reports return different values for ReportContext"; 135 first = false; 136 last_ret = new_ret; 137 } 138 (void)first; 139 return last_ret; 140 } 141 142 void ReportRuns(const std::vector<Run>& report) { 143 for (auto rep : reporters_) rep->ReportRuns(report); 144 } 145 void Finalize() { 146 for (auto rep : reporters_) rep->Finalize(); 147 } 148 149 private: 150 std::vector<benchmark::BenchmarkReporter *> reporters_; 151 }; 152 } 153 154 } // end namespace internal 155 156 // ========================================================================= // 157 // -------------------------- Results checking ----------------------------- // 158 // ========================================================================= // 159 160 namespace internal { 161 162 // Utility class to manage subscribers for checking benchmark results. 163 // It works by parsing the CSV output to read the results. 164 class ResultsChecker { 165 public: 166 167 struct PatternAndFn : public TestCase { // reusing TestCase for its regexes 168 PatternAndFn(const std::string& rx, ResultsCheckFn fn_) 169 : TestCase(rx), fn(fn_) {} 170 ResultsCheckFn fn; 171 }; 172 173 std::vector< PatternAndFn > check_patterns; 174 std::vector< Results > results; 175 std::vector< std::string > field_names; 176 177 void Add(const std::string& entry_pattern, ResultsCheckFn fn); 178 179 void CheckResults(std::stringstream& output); 180 181 private: 182 183 void SetHeader_(const std::string& csv_header); 184 void SetValues_(const std::string& entry_csv_line); 185 186 std::vector< std::string > SplitCsv_(const std::string& line); 187 188 }; 189 190 // store the static ResultsChecker in a function to prevent initialization 191 // order problems 192 ResultsChecker& GetResultsChecker() { 193 static ResultsChecker rc; 194 return rc; 195 } 196 197 // add a results checker for a benchmark 198 void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) { 199 check_patterns.emplace_back(entry_pattern, fn); 200 } 201 202 // check the results of all subscribed benchmarks 203 void ResultsChecker::CheckResults(std::stringstream& output) { 204 // first reset the stream to the start 205 { 206 auto start = std::ios::streampos(0); 207 // clear before calling tellg() 208 output.clear(); 209 // seek to zero only when needed 210 if(output.tellg() > start) output.seekg(start); 211 // and just in case 212 output.clear(); 213 } 214 // now go over every line and publish it to the ResultsChecker 215 std::string line; 216 bool on_first = true; 217 while (output.eof() == false) { 218 CHECK(output.good()); 219 std::getline(output, line); 220 if (on_first) { 221 SetHeader_(line); // this is important 222 on_first = false; 223 continue; 224 } 225 SetValues_(line); 226 } 227 // finally we can call the subscribed check functions 228 for(const auto& p : check_patterns) { 229 VLOG(2) << "--------------------------------\n"; 230 VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; 231 for(const auto& r : results) { 232 if(!p.regex->Match(r.name)) { 233 VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; 234 continue; 235 } else { 236 VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; 237 } 238 VLOG(1) << "Checking results of " << r.name << ": ... \n"; 239 p.fn(r); 240 VLOG(1) << "Checking results of " << r.name << ": OK.\n"; 241 } 242 } 243 } 244 245 // prepare for the names in this header 246 void ResultsChecker::SetHeader_(const std::string& csv_header) { 247 field_names = SplitCsv_(csv_header); 248 } 249 250 // set the values for a benchmark 251 void ResultsChecker::SetValues_(const std::string& entry_csv_line) { 252 if(entry_csv_line.empty()) return; // some lines are empty 253 CHECK(!field_names.empty()); 254 auto vals = SplitCsv_(entry_csv_line); 255 CHECK_EQ(vals.size(), field_names.size()); 256 results.emplace_back(vals[0]); // vals[0] is the benchmark name 257 auto &entry = results.back(); 258 for (size_t i = 1, e = vals.size(); i < e; ++i) { 259 entry.values[field_names[i]] = vals[i]; 260 } 261 } 262 263 // a quick'n'dirty csv splitter (eliminating quotes) 264 std::vector< std::string > ResultsChecker::SplitCsv_(const std::string& line) { 265 std::vector< std::string > out; 266 if(line.empty()) return out; 267 if(!field_names.empty()) out.reserve(field_names.size()); 268 size_t prev = 0, pos = line.find_first_of(','), curr = pos; 269 while(pos != line.npos) { 270 CHECK(curr > 0); 271 if(line[prev] == '"') ++prev; 272 if(line[curr-1] == '"') --curr; 273 out.push_back(line.substr(prev, curr-prev)); 274 prev = pos + 1; 275 pos = line.find_first_of(',', pos + 1); 276 curr = pos; 277 } 278 curr = line.size(); 279 if(line[prev] == '"') ++prev; 280 if(line[curr-1] == '"') --curr; 281 out.push_back(line.substr(prev, curr-prev)); 282 return out; 283 } 284 285 } // end namespace internal 286 287 size_t AddChecker(const char* bm_name, ResultsCheckFn fn) 288 { 289 auto &rc = internal::GetResultsChecker(); 290 rc.Add(bm_name, fn); 291 return rc.results.size(); 292 } 293 294 int Results::NumThreads() const { 295 auto pos = name.find("/threads:"); 296 if(pos == name.npos) return 1; 297 auto end = name.find('/', pos + 9); 298 std::stringstream ss; 299 ss << name.substr(pos + 9, end); 300 int num = 1; 301 ss >> num; 302 CHECK(!ss.fail()); 303 return num; 304 } 305 306 double Results::GetTime(BenchmarkTime which) const { 307 CHECK(which == kCpuTime || which == kRealTime); 308 const char *which_str = which == kCpuTime ? "cpu_time" : "real_time"; 309 double val = GetAs< double >(which_str); 310 auto unit = Get("time_unit"); 311 CHECK(unit); 312 if(*unit == "ns") { 313 return val * 1.e-9; 314 } else if(*unit == "us") { 315 return val * 1.e-6; 316 } else if(*unit == "ms") { 317 return val * 1.e-3; 318 } else if(*unit == "s") { 319 return val; 320 } else { 321 CHECK(1 == 0) << "unknown time unit: " << *unit; 322 return 0; 323 } 324 } 325 326 // ========================================================================= // 327 // -------------------------- Public API Definitions------------------------ // 328 // ========================================================================= // 329 330 TestCase::TestCase(std::string re, int rule) 331 : regex_str(std::move(re)), 332 match_rule(rule), 333 substituted_regex(internal::PerformSubstitutions(regex_str)), 334 regex(std::make_shared<benchmark::Regex>()) { 335 std::string err_str; 336 regex->Init(substituted_regex,& err_str); 337 CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex 338 << "\"" 339 << "\n originally \"" << regex_str << "\"" 340 << "\n got error: " << err_str; 341 } 342 343 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) { 344 auto& L = internal::GetTestCaseList(ID); 345 L.insert(L.end(), il); 346 return 0; 347 } 348 349 int SetSubstitutions( 350 std::initializer_list<std::pair<std::string, std::string>> il) { 351 auto& subs = internal::GetSubstitutions(); 352 for (auto KV : il) { 353 bool exists = false; 354 KV.second = internal::PerformSubstitutions(KV.second); 355 for (auto& EKV : subs) { 356 if (EKV.first == KV.first) { 357 EKV.second = std::move(KV.second); 358 exists = true; 359 break; 360 } 361 } 362 if (!exists) subs.push_back(std::move(KV)); 363 } 364 return 0; 365 } 366 367 void RunOutputTests(int argc, char* argv[]) { 368 using internal::GetTestCaseList; 369 benchmark::Initialize(&argc, argv); 370 auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/true); 371 benchmark::ConsoleReporter CR(options); 372 benchmark::JSONReporter JR; 373 benchmark::CSVReporter CSVR; 374 struct ReporterTest { 375 const char* name; 376 std::vector<TestCase>& output_cases; 377 std::vector<TestCase>& error_cases; 378 benchmark::BenchmarkReporter& reporter; 379 std::stringstream out_stream; 380 std::stringstream err_stream; 381 382 ReporterTest(const char* n, std::vector<TestCase>& out_tc, 383 std::vector<TestCase>& err_tc, 384 benchmark::BenchmarkReporter& br) 385 : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { 386 reporter.SetOutputStream(&out_stream); 387 reporter.SetErrorStream(&err_stream); 388 } 389 } TestCases[] = { 390 {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), 391 GetTestCaseList(TC_ConsoleErr), CR}, 392 {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), 393 JR}, 394 {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), 395 CSVR}, 396 }; 397 398 // Create the test reporter and run the benchmarks. 399 std::cout << "Running benchmarks...\n"; 400 internal::TestReporter test_rep({&CR, &JR, &CSVR}); 401 benchmark::RunSpecifiedBenchmarks(&test_rep); 402 403 for (auto& rep_test : TestCases) { 404 std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; 405 std::string banner(msg.size() - 1, '-'); 406 std::cout << banner << msg << banner << "\n"; 407 408 std::cerr << rep_test.err_stream.str(); 409 std::cout << rep_test.out_stream.str(); 410 411 internal::CheckCases(rep_test.error_cases, rep_test.err_stream); 412 internal::CheckCases(rep_test.output_cases, rep_test.out_stream); 413 414 std::cout << "\n"; 415 } 416 417 // now that we know the output is as expected, we can dispatch 418 // the checks to subscribees. 419 auto &csv = TestCases[2]; 420 // would use == but gcc spits a warning 421 CHECK(std::strcmp(csv.name, "CSVReporter") == 0); 422 internal::GetResultsChecker().CheckResults(csv.out_stream); 423 } 424