1 2 #undef NDEBUG 3 4 #include "benchmark/benchmark.h" 5 #include "output_test.h" 6 7 // ========================================================================= // 8 // ---------------------- Testing Prologue Output -------------------------- // 9 // ========================================================================= // 10 11 // clang-format off 12 13 ADD_CASES(TC_ConsoleOut, 14 {{"^[-]+$", MR_Next}, 15 {"^Benchmark %s Time %s CPU %s Iterations UserCounters...$", MR_Next}, 16 {"^[-]+$", MR_Next}}); 17 ADD_CASES(TC_CSVOut, {{"%csv_header,\"bar\",\"foo\""}}); 18 19 // clang-format on 20 21 // ========================================================================= // 22 // ------------------------- Simple Counters Output ------------------------ // 23 // ========================================================================= // 24 25 void BM_Counters_Simple(benchmark::State& state) { 26 for (auto _ : state) { 27 } 28 state.counters["foo"] = 1; 29 state.counters["bar"] = 2 * (double)state.iterations(); 30 } 31 BENCHMARK(BM_Counters_Simple); 32 ADD_CASES(TC_ConsoleOut, 33 {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); 34 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, 35 {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, 36 {"\"run_type\": \"iteration\",$", MR_Next}, 37 {"\"iterations\": %int,$", MR_Next}, 38 {"\"real_time\": %float,$", MR_Next}, 39 {"\"cpu_time\": %float,$", MR_Next}, 40 {"\"time_unit\": \"ns\",$", MR_Next}, 41 {"\"bar\": %float,$", MR_Next}, 42 {"\"foo\": %float$", MR_Next}, 43 {"}", MR_Next}}); 44 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}}); 45 // VS2013 does not allow this function to be passed as a lambda argument 46 // to CHECK_BENCHMARK_RESULTS() 47 void CheckSimple(Results const& e) { 48 double its = e.NumIterations(); 49 CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); 50 // check that the value of bar is within 0.1% of the expected value 51 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001); 52 } 53 CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple); 54 55 // ========================================================================= // 56 // --------------------- Counters+Items+Bytes/s Output --------------------- // 57 // ========================================================================= // 58 59 namespace { 60 int num_calls1 = 0; 61 } 62 void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { 63 for (auto _ : state) { 64 } 65 state.counters["foo"] = 1; 66 state.counters["bar"] = ++num_calls1; 67 state.SetBytesProcessed(364); 68 state.SetItemsProcessed(150); 69 } 70 BENCHMARK(BM_Counters_WithBytesAndItemsPSec); 71 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " 72 "bar=%hrfloat bytes_per_second=%hrfloat/s " 73 "foo=%hrfloat items_per_second=%hrfloat/s$"}}); 74 ADD_CASES(TC_JSONOut, 75 {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, 76 {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, 77 {"\"run_type\": \"iteration\",$", MR_Next}, 78 {"\"iterations\": %int,$", MR_Next}, 79 {"\"real_time\": %float,$", MR_Next}, 80 {"\"cpu_time\": %float,$", MR_Next}, 81 {"\"time_unit\": \"ns\",$", MR_Next}, 82 {"\"bar\": %float,$", MR_Next}, 83 {"\"bytes_per_second\": %float,$", MR_Next}, 84 {"\"foo\": %float,$", MR_Next}, 85 {"\"items_per_second\": %float$", MR_Next}, 86 {"}", MR_Next}}); 87 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," 88 "%csv_bytes_items_report,%float,%float$"}}); 89 // VS2013 does not allow this function to be passed as a lambda argument 90 // to CHECK_BENCHMARK_RESULTS() 91 void CheckBytesAndItemsPSec(Results const& e) { 92 double t = e.DurationCPUTime(); // this (and not real time) is the time used 93 CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); 94 CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1); 95 // check that the values are within 0.1% of the expected values 96 CHECK_FLOAT_RESULT_VALUE(e, "bytes_per_second", EQ, 364. / t, 0.001); 97 CHECK_FLOAT_RESULT_VALUE(e, "items_per_second", EQ, 150. / t, 0.001); 98 } 99 CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", 100 &CheckBytesAndItemsPSec); 101 102 // ========================================================================= // 103 // ------------------------- Rate Counters Output -------------------------- // 104 // ========================================================================= // 105 106 void BM_Counters_Rate(benchmark::State& state) { 107 for (auto _ : state) { 108 } 109 namespace bm = benchmark; 110 state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; 111 state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate}; 112 } 113 BENCHMARK(BM_Counters_Rate); 114 ADD_CASES( 115 TC_ConsoleOut, 116 {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); 117 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, 118 {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, 119 {"\"run_type\": \"iteration\",$", MR_Next}, 120 {"\"iterations\": %int,$", MR_Next}, 121 {"\"real_time\": %float,$", MR_Next}, 122 {"\"cpu_time\": %float,$", MR_Next}, 123 {"\"time_unit\": \"ns\",$", MR_Next}, 124 {"\"bar\": %float,$", MR_Next}, 125 {"\"foo\": %float$", MR_Next}, 126 {"}", MR_Next}}); 127 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}}); 128 // VS2013 does not allow this function to be passed as a lambda argument 129 // to CHECK_BENCHMARK_RESULTS() 130 void CheckRate(Results const& e) { 131 double t = e.DurationCPUTime(); // this (and not real time) is the time used 132 // check that the values are within 0.1% of the expected values 133 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / t, 0.001); 134 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / t, 0.001); 135 } 136 CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); 137 138 // ========================================================================= // 139 // ------------------------- Thread Counters Output ------------------------ // 140 // ========================================================================= // 141 142 void BM_Counters_Threads(benchmark::State& state) { 143 for (auto _ : state) { 144 } 145 state.counters["foo"] = 1; 146 state.counters["bar"] = 2; 147 } 148 BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); 149 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " 150 "bar=%hrfloat foo=%hrfloat$"}}); 151 ADD_CASES(TC_JSONOut, 152 {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, 153 {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, 154 {"\"run_type\": \"iteration\",$", MR_Next}, 155 {"\"iterations\": %int,$", MR_Next}, 156 {"\"real_time\": %float,$", MR_Next}, 157 {"\"cpu_time\": %float,$", MR_Next}, 158 {"\"time_unit\": \"ns\",$", MR_Next}, 159 {"\"bar\": %float,$", MR_Next}, 160 {"\"foo\": %float$", MR_Next}, 161 {"}", MR_Next}}); 162 ADD_CASES( 163 TC_CSVOut, 164 {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}}); 165 // VS2013 does not allow this function to be passed as a lambda argument 166 // to CHECK_BENCHMARK_RESULTS() 167 void CheckThreads(Results const& e) { 168 CHECK_COUNTER_VALUE(e, int, "foo", EQ, e.NumThreads()); 169 CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2 * e.NumThreads()); 170 } 171 CHECK_BENCHMARK_RESULTS("BM_Counters_Threads/threads:%int", &CheckThreads); 172 173 // ========================================================================= // 174 // ---------------------- ThreadAvg Counters Output ------------------------ // 175 // ========================================================================= // 176 177 void BM_Counters_AvgThreads(benchmark::State& state) { 178 for (auto _ : state) { 179 } 180 namespace bm = benchmark; 181 state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreads}; 182 state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads}; 183 } 184 BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); 185 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " 186 "%console_report bar=%hrfloat foo=%hrfloat$"}}); 187 ADD_CASES(TC_JSONOut, 188 {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, 189 {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, 190 {"\"run_type\": \"iteration\",$", MR_Next}, 191 {"\"iterations\": %int,$", MR_Next}, 192 {"\"real_time\": %float,$", MR_Next}, 193 {"\"cpu_time\": %float,$", MR_Next}, 194 {"\"time_unit\": \"ns\",$", MR_Next}, 195 {"\"bar\": %float,$", MR_Next}, 196 {"\"foo\": %float$", MR_Next}, 197 {"}", MR_Next}}); 198 ADD_CASES( 199 TC_CSVOut, 200 {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}}); 201 // VS2013 does not allow this function to be passed as a lambda argument 202 // to CHECK_BENCHMARK_RESULTS() 203 void CheckAvgThreads(Results const& e) { 204 CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); 205 CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2); 206 } 207 CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", 208 &CheckAvgThreads); 209 210 // ========================================================================= // 211 // ---------------------- ThreadAvg Counters Output ------------------------ // 212 // ========================================================================= // 213 214 void BM_Counters_AvgThreadsRate(benchmark::State& state) { 215 for (auto _ : state) { 216 } 217 namespace bm = benchmark; 218 state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; 219 state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate}; 220 } 221 BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8); 222 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " 223 "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); 224 ADD_CASES(TC_JSONOut, 225 {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, 226 {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", 227 MR_Next}, 228 {"\"run_type\": \"iteration\",$", MR_Next}, 229 {"\"iterations\": %int,$", MR_Next}, 230 {"\"real_time\": %float,$", MR_Next}, 231 {"\"cpu_time\": %float,$", MR_Next}, 232 {"\"time_unit\": \"ns\",$", MR_Next}, 233 {"\"bar\": %float,$", MR_Next}, 234 {"\"foo\": %float$", MR_Next}, 235 {"}", MR_Next}}); 236 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/" 237 "threads:%int\",%csv_report,%float,%float$"}}); 238 // VS2013 does not allow this function to be passed as a lambda argument 239 // to CHECK_BENCHMARK_RESULTS() 240 void CheckAvgThreadsRate(Results const& e) { 241 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / e.DurationCPUTime(), 0.001); 242 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / e.DurationCPUTime(), 0.001); 243 } 244 CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int", 245 &CheckAvgThreadsRate); 246 247 // ========================================================================= // 248 // ------------------- IterationInvariant Counters Output ------------------ // 249 // ========================================================================= // 250 251 void BM_Counters_IterationInvariant(benchmark::State& state) { 252 for (auto _ : state) { 253 } 254 namespace bm = benchmark; 255 state.counters["foo"] = bm::Counter{1, bm::Counter::kIsIterationInvariant}; 256 state.counters["bar"] = bm::Counter{2, bm::Counter::kIsIterationInvariant}; 257 } 258 BENCHMARK(BM_Counters_IterationInvariant); 259 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " 260 "bar=%hrfloat foo=%hrfloat$"}}); 261 ADD_CASES(TC_JSONOut, 262 {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, 263 {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, 264 {"\"run_type\": \"iteration\",$", MR_Next}, 265 {"\"iterations\": %int,$", MR_Next}, 266 {"\"real_time\": %float,$", MR_Next}, 267 {"\"cpu_time\": %float,$", MR_Next}, 268 {"\"time_unit\": \"ns\",$", MR_Next}, 269 {"\"bar\": %float,$", MR_Next}, 270 {"\"foo\": %float$", MR_Next}, 271 {"}", MR_Next}}); 272 ADD_CASES(TC_CSVOut, 273 {{"^\"BM_Counters_IterationInvariant\",%csv_report,%float,%float$"}}); 274 // VS2013 does not allow this function to be passed as a lambda argument 275 // to CHECK_BENCHMARK_RESULTS() 276 void CheckIterationInvariant(Results const& e) { 277 double its = e.NumIterations(); 278 // check that the values are within 0.1% of the expected value 279 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its, 0.001); 280 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001); 281 } 282 CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant", 283 &CheckIterationInvariant); 284 285 // ========================================================================= // 286 // ----------------- IterationInvariantRate Counters Output ---------------- // 287 // ========================================================================= // 288 289 void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { 290 for (auto _ : state) { 291 } 292 namespace bm = benchmark; 293 state.counters["foo"] = 294 bm::Counter{1, bm::Counter::kIsIterationInvariantRate}; 295 state.counters["bar"] = 296 bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kIsIterationInvariant}; 297 } 298 BENCHMARK(BM_Counters_kIsIterationInvariantRate); 299 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " 300 "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); 301 ADD_CASES(TC_JSONOut, 302 {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, 303 {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", 304 MR_Next}, 305 {"\"run_type\": \"iteration\",$", MR_Next}, 306 {"\"iterations\": %int,$", MR_Next}, 307 {"\"real_time\": %float,$", MR_Next}, 308 {"\"cpu_time\": %float,$", MR_Next}, 309 {"\"time_unit\": \"ns\",$", MR_Next}, 310 {"\"bar\": %float,$", MR_Next}, 311 {"\"foo\": %float$", MR_Next}, 312 {"}", MR_Next}}); 313 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kIsIterationInvariantRate\",%csv_report," 314 "%float,%float$"}}); 315 // VS2013 does not allow this function to be passed as a lambda argument 316 // to CHECK_BENCHMARK_RESULTS() 317 void CheckIsIterationInvariantRate(Results const& e) { 318 double its = e.NumIterations(); 319 double t = e.DurationCPUTime(); // this (and not real time) is the time used 320 // check that the values are within 0.1% of the expected values 321 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its * 1. / t, 0.001); 322 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, its * 2. / t, 0.001); 323 } 324 CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate", 325 &CheckIsIterationInvariantRate); 326 327 // ========================================================================= // 328 // ------------------- AvgIterations Counters Output ------------------ // 329 // ========================================================================= // 330 331 void BM_Counters_AvgIterations(benchmark::State& state) { 332 for (auto _ : state) { 333 } 334 namespace bm = benchmark; 335 state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterations}; 336 state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgIterations}; 337 } 338 BENCHMARK(BM_Counters_AvgIterations); 339 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " 340 "bar=%hrfloat foo=%hrfloat$"}}); 341 ADD_CASES(TC_JSONOut, 342 {{"\"name\": \"BM_Counters_AvgIterations\",$"}, 343 {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, 344 {"\"run_type\": \"iteration\",$", MR_Next}, 345 {"\"iterations\": %int,$", MR_Next}, 346 {"\"real_time\": %float,$", MR_Next}, 347 {"\"cpu_time\": %float,$", MR_Next}, 348 {"\"time_unit\": \"ns\",$", MR_Next}, 349 {"\"bar\": %float,$", MR_Next}, 350 {"\"foo\": %float$", MR_Next}, 351 {"}", MR_Next}}); 352 ADD_CASES(TC_CSVOut, 353 {{"^\"BM_Counters_AvgIterations\",%csv_report,%float,%float$"}}); 354 // VS2013 does not allow this function to be passed as a lambda argument 355 // to CHECK_BENCHMARK_RESULTS() 356 void CheckAvgIterations(Results const& e) { 357 double its = e.NumIterations(); 358 // check that the values are within 0.1% of the expected value 359 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its, 0.001); 360 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its, 0.001); 361 } 362 CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations); 363 364 // ========================================================================= // 365 // ----------------- AvgIterationsRate Counters Output ---------------- // 366 // ========================================================================= // 367 368 void BM_Counters_kAvgIterationsRate(benchmark::State& state) { 369 for (auto _ : state) { 370 } 371 namespace bm = benchmark; 372 state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; 373 state.counters["bar"] = 374 bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kAvgIterations}; 375 } 376 BENCHMARK(BM_Counters_kAvgIterationsRate); 377 ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " 378 "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); 379 ADD_CASES(TC_JSONOut, 380 {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, 381 {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, 382 {"\"run_type\": \"iteration\",$", MR_Next}, 383 {"\"iterations\": %int,$", MR_Next}, 384 {"\"real_time\": %float,$", MR_Next}, 385 {"\"cpu_time\": %float,$", MR_Next}, 386 {"\"time_unit\": \"ns\",$", MR_Next}, 387 {"\"bar\": %float,$", MR_Next}, 388 {"\"foo\": %float$", MR_Next}, 389 {"}", MR_Next}}); 390 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kAvgIterationsRate\",%csv_report," 391 "%float,%float$"}}); 392 // VS2013 does not allow this function to be passed as a lambda argument 393 // to CHECK_BENCHMARK_RESULTS() 394 void CheckAvgIterationsRate(Results const& e) { 395 double its = e.NumIterations(); 396 double t = e.DurationCPUTime(); // this (and not real time) is the time used 397 // check that the values are within 0.1% of the expected values 398 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its / t, 0.001); 399 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its / t, 0.001); 400 } 401 CHECK_BENCHMARK_RESULTS("BM_Counters_kAvgIterationsRate", 402 &CheckAvgIterationsRate); 403 404 // ========================================================================= // 405 // --------------------------- TEST CASES END ------------------------------ // 406 // ========================================================================= // 407 408 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } 409
