1 // Copyright 2015 Google Inc. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Support for registering benchmarks for functions. 16 17 /* Example usage: 18 // Define a function that executes the code to be measured a 19 // specified number of times: 20 static void BM_StringCreation(benchmark::State& state) { 21 for (auto _ : state) 22 std::string empty_string; 23 } 24 25 // Register the function as a benchmark 26 BENCHMARK(BM_StringCreation); 27 28 // Define another benchmark 29 static void BM_StringCopy(benchmark::State& state) { 30 std::string x = "hello"; 31 for (auto _ : state) 32 std::string copy(x); 33 } 34 BENCHMARK(BM_StringCopy); 35 36 // Augment the main() program to invoke benchmarks if specified 37 // via the --benchmarks command line flag. E.g., 38 // my_unittest --benchmark_filter=all 39 // my_unittest --benchmark_filter=BM_StringCreation 40 // my_unittest --benchmark_filter=String 41 // my_unittest --benchmark_filter='Copy|Creation' 42 int main(int argc, char** argv) { 43 benchmark::Initialize(&argc, argv); 44 benchmark::RunSpecifiedBenchmarks(); 45 return 0; 46 } 47 48 // Sometimes a family of microbenchmarks can be implemented with 49 // just one routine that takes an extra argument to specify which 50 // one of the family of benchmarks to run. For example, the following 51 // code defines a family of microbenchmarks for measuring the speed 52 // of memcpy() calls of different lengths: 53 54 static void BM_memcpy(benchmark::State& state) { 55 char* src = new char[state.range(0)]; char* dst = new char[state.range(0)]; 56 memset(src, 'x', state.range(0)); 57 for (auto _ : state) 58 memcpy(dst, src, state.range(0)); 59 state.SetBytesProcessed(int64_t(state.iterations()) * 60 int64_t(state.range(0))); 61 delete[] src; delete[] dst; 62 } 63 BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); 64 65 // The preceding code is quite repetitive, and can be replaced with the 66 // following short-hand. The following invocation will pick a few 67 // appropriate arguments in the specified range and will generate a 68 // microbenchmark for each such argument. 69 BENCHMARK(BM_memcpy)->Range(8, 8<<10); 70 71 // You might have a microbenchmark that depends on two inputs. For 72 // example, the following code defines a family of microbenchmarks for 73 // measuring the speed of set insertion. 74 static void BM_SetInsert(benchmark::State& state) { 75 set<int> data; 76 for (auto _ : state) { 77 state.PauseTiming(); 78 data = ConstructRandomSet(state.range(0)); 79 state.ResumeTiming(); 80 for (int j = 0; j < state.range(1); ++j) 81 data.insert(RandomNumber()); 82 } 83 } 84 BENCHMARK(BM_SetInsert) 85 ->Args({1<<10, 128}) 86 ->Args({2<<10, 128}) 87 ->Args({4<<10, 128}) 88 ->Args({8<<10, 128}) 89 ->Args({1<<10, 512}) 90 ->Args({2<<10, 512}) 91 ->Args({4<<10, 512}) 92 ->Args({8<<10, 512}); 93 94 // The preceding code is quite repetitive, and can be replaced with 95 // the following short-hand. The following macro will pick a few 96 // appropriate arguments in the product of the two specified ranges 97 // and will generate a microbenchmark for each such pair. 98 BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); 99 100 // For more complex patterns of inputs, passing a custom function 101 // to Apply allows programmatic specification of an 102 // arbitrary set of arguments to run the microbenchmark on. 103 // The following example enumerates a dense range on 104 // one parameter, and a sparse range on the second. 105 static void CustomArguments(benchmark::internal::Benchmark* b) { 106 for (int i = 0; i <= 10; ++i) 107 for (int j = 32; j <= 1024*1024; j *= 8) 108 b->Args({i, j}); 109 } 110 BENCHMARK(BM_SetInsert)->Apply(CustomArguments); 111 112 // Templated microbenchmarks work the same way: 113 // Produce then consume 'size' messages 'iters' times 114 // Measures throughput in the absence of multiprogramming. 115 template <class Q> int BM_Sequential(benchmark::State& state) { 116 Q q; 117 typename Q::value_type v; 118 for (auto _ : state) { 119 for (int i = state.range(0); i--; ) 120 q.push(v); 121 for (int e = state.range(0); e--; ) 122 q.Wait(&v); 123 } 124 // actually messages, not bytes: 125 state.SetBytesProcessed( 126 static_cast<int64_t>(state.iterations())*state.range(0)); 127 } 128 BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue<int>)->Range(1<<0, 1<<10); 129 130 Use `Benchmark::MinTime(double t)` to set the minimum time used to run the 131 benchmark. This option overrides the `benchmark_min_time` flag. 132 133 void BM_test(benchmark::State& state) { 134 ... body ... 135 } 136 BENCHMARK(BM_test)->MinTime(2.0); // Run for at least 2 seconds. 137 138 In a multithreaded test, it is guaranteed that none of the threads will start 139 until all have reached the loop start, and all will have finished before any 140 thread exits the loop body. As such, any global setup or teardown you want to 141 do can be wrapped in a check against the thread index: 142 143 static void BM_MultiThreaded(benchmark::State& state) { 144 if (state.thread_index == 0) { 145 // Setup code here. 146 } 147 for (auto _ : state) { 148 // Run the test as normal. 149 } 150 if (state.thread_index == 0) { 151 // Teardown code here. 152 } 153 } 154 BENCHMARK(BM_MultiThreaded)->Threads(4); 155 156 157 If a benchmark runs a few milliseconds it may be hard to visually compare the 158 measured times, since the output data is given in nanoseconds per default. In 159 order to manually set the time unit, you can specify it manually: 160 161 BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); 162 */ 163 164 #ifndef BENCHMARK_BENCHMARK_H_ 165 #define BENCHMARK_BENCHMARK_H_ 166 167 168 // The _MSVC_LANG check should detect Visual Studio 2015 Update 3 and newer. 169 #if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) 170 #define BENCHMARK_HAS_CXX11 171 #endif 172 173 #include <stdint.h> 174 175 #include <cassert> 176 #include <cstddef> 177 #include <iosfwd> 178 #include <string> 179 #include <vector> 180 #include <map> 181 #include <set> 182 183 #if defined(BENCHMARK_HAS_CXX11) 184 #include <type_traits> 185 #include <initializer_list> 186 #include <utility> 187 #endif 188 189 #if defined(_MSC_VER) 190 #include <intrin.h> // for _ReadWriteBarrier 191 #endif 192 193 #ifndef BENCHMARK_HAS_CXX11 194 #define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ 195 TypeName(const TypeName&); \ 196 TypeName& operator=(const TypeName&) 197 #else 198 #define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ 199 TypeName(const TypeName&) = delete; \ 200 TypeName& operator=(const TypeName&) = delete 201 #endif 202 203 #if defined(__GNUC__) 204 #define BENCHMARK_UNUSED __attribute__((unused)) 205 #define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) 206 #define BENCHMARK_NOEXCEPT noexcept 207 #define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) 208 #elif defined(_MSC_VER) && !defined(__clang__) 209 #define BENCHMARK_UNUSED 210 #define BENCHMARK_ALWAYS_INLINE __forceinline 211 #if _MSC_VER >= 1900 212 #define BENCHMARK_NOEXCEPT noexcept 213 #define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) 214 #else 215 #define BENCHMARK_NOEXCEPT 216 #define BENCHMARK_NOEXCEPT_OP(x) 217 #endif 218 #define __func__ __FUNCTION__ 219 #else 220 #define BENCHMARK_UNUSED 221 #define BENCHMARK_ALWAYS_INLINE 222 #define BENCHMARK_NOEXCEPT 223 #define BENCHMARK_NOEXCEPT_OP(x) 224 #endif 225 226 #define BENCHMARK_INTERNAL_TOSTRING2(x) #x 227 #define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) 228 229 #if defined(__GNUC__) 230 #define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) 231 #define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) 232 #else 233 #define BENCHMARK_BUILTIN_EXPECT(x, y) x 234 #define BENCHMARK_DEPRECATED_MSG(msg) 235 #define BENCHMARK_WARNING_MSG(msg) __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING(__LINE__) ") : warning note: " msg)) 236 #endif 237 238 #if defined(__GNUC__) && !defined(__clang__) 239 #define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 240 #endif 241 242 namespace benchmark { 243 class BenchmarkReporter; 244 245 void Initialize(int* argc, char** argv); 246 247 // Report to stdout all arguments in 'argv' as unrecognized except the first. 248 // Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). 249 bool ReportUnrecognizedArguments(int argc, char** argv); 250 251 // Generate a list of benchmarks matching the specified --benchmark_filter flag 252 // and if --benchmark_list_tests is specified return after printing the name 253 // of each matching benchmark. Otherwise run each matching benchmark and 254 // report the results. 255 // 256 // The second and third overload use the specified 'console_reporter' and 257 // 'file_reporter' respectively. 'file_reporter' will write to the file 258 // specified 259 // by '--benchmark_output'. If '--benchmark_output' is not given the 260 // 'file_reporter' is ignored. 261 // 262 // RETURNS: The number of matching benchmarks. 263 size_t RunSpecifiedBenchmarks(); 264 size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter); 265 size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, 266 BenchmarkReporter* file_reporter); 267 268 // If this routine is called, peak memory allocation past this point in the 269 // benchmark is reported at the end of the benchmark report line. (It is 270 // computed by running the benchmark once with a single iteration and a memory 271 // tracer.) 272 // TODO(dominic) 273 // void MemoryUsage(); 274 275 namespace internal { 276 class Benchmark; 277 class BenchmarkImp; 278 class BenchmarkFamilies; 279 280 void UseCharPointer(char const volatile*); 281 282 // Take ownership of the pointer and register the benchmark. Return the 283 // registered benchmark. 284 Benchmark* RegisterBenchmarkInternal(Benchmark*); 285 286 // Ensure that the standard streams are properly initialized in every TU. 287 int InitializeStreams(); 288 BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); 289 290 } // namespace internal 291 292 293 #if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \ 294 defined(EMSCRIPTN) 295 # define BENCHMARK_HAS_NO_INLINE_ASSEMBLY 296 #endif 297 298 299 // The DoNotOptimize(...) function can be used to prevent a value or 300 // expression from being optimized away by the compiler. This function is 301 // intended to add little to no overhead. 302 // See: https://youtu.be/nXaxk27zwlk?t=2441 303 #ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY 304 template <class Tp> 305 inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { 306 // Clang doesn't like the 'X' constraint on `value` and certain GCC versions 307 // don't like the 'g' constraint. Attempt to placate them both. 308 #if defined(__clang__) 309 asm volatile("" : : "g"(value) : "memory"); 310 #else 311 asm volatile("" : : "i,r,m"(value) : "memory"); 312 #endif 313 } 314 // Force the compiler to flush pending writes to global memory. Acts as an 315 // effective read/write barrier 316 inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { 317 asm volatile("" : : : "memory"); 318 } 319 #elif defined(_MSC_VER) 320 template <class Tp> 321 inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { 322 internal::UseCharPointer(&reinterpret_cast<char const volatile&>(value)); 323 _ReadWriteBarrier(); 324 } 325 326 inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { 327 _ReadWriteBarrier(); 328 } 329 #else 330 template <class Tp> 331 inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { 332 internal::UseCharPointer(&reinterpret_cast<char const volatile&>(value)); 333 } 334 // FIXME Add ClobberMemory() for non-gnu and non-msvc compilers 335 #endif 336 337 338 339 // This class is used for user-defined counters. 340 class Counter { 341 public: 342 343 enum Flags { 344 kDefaults = 0, 345 // Mark the counter as a rate. It will be presented divided 346 // by the duration of the benchmark. 347 kIsRate = 1, 348 // Mark the counter as a thread-average quantity. It will be 349 // presented divided by the number of threads. 350 kAvgThreads = 2, 351 // Mark the counter as a thread-average rate. See above. 352 kAvgThreadsRate = kIsRate|kAvgThreads 353 }; 354 355 double value; 356 Flags flags; 357 358 BENCHMARK_ALWAYS_INLINE 359 Counter(double v = 0., Flags f = kDefaults) : value(v), flags(f) {} 360 361 BENCHMARK_ALWAYS_INLINE operator double const& () const { return value; } 362 BENCHMARK_ALWAYS_INLINE operator double & () { return value; } 363 364 }; 365 366 // This is the container for the user-defined counters. 367 typedef std::map<std::string, Counter> UserCounters; 368 369 370 // TimeUnit is passed to a benchmark in order to specify the order of magnitude 371 // for the measured time. 372 enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond }; 373 374 // BigO is passed to a benchmark in order to specify the asymptotic 375 // computational 376 // complexity for the benchmark. In case oAuto is selected, complexity will be 377 // calculated automatically to the best fit. 378 enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; 379 380 // BigOFunc is passed to a benchmark in order to specify the asymptotic 381 // computational complexity for the benchmark. 382 typedef double(BigOFunc)(int); 383 384 // StatisticsFunc is passed to a benchmark in order to compute some descriptive 385 // statistics over all the measurements of some type 386 typedef double(StatisticsFunc)(const std::vector<double>&); 387 388 struct Statistics { 389 std::string name_; 390 StatisticsFunc* compute_; 391 392 Statistics(std::string name, StatisticsFunc* compute) 393 : name_(name), compute_(compute) {} 394 }; 395 396 namespace internal { 397 class ThreadTimer; 398 class ThreadManager; 399 400 enum ReportMode 401 #if defined(BENCHMARK_HAS_CXX11) 402 : unsigned 403 #else 404 #endif 405 { 406 RM_Unspecified, // The mode has not been manually specified 407 RM_Default, // The mode is user-specified as default. 408 RM_ReportAggregatesOnly 409 }; 410 } // namespace internal 411 412 // State is passed to a running Benchmark and contains state for the 413 // benchmark to use. 414 class State { 415 public: 416 struct StateIterator; 417 friend struct StateIterator; 418 419 // Returns iterators used to run each iteration of a benchmark using a 420 // C++11 ranged-based for loop. These functions should not be called directly. 421 // 422 // REQUIRES: The benchmark has not started running yet. Neither begin nor end 423 // have been called previously. 424 // 425 // NOTE: KeepRunning may not be used after calling either of these functions. 426 BENCHMARK_ALWAYS_INLINE StateIterator begin(); 427 BENCHMARK_ALWAYS_INLINE StateIterator end(); 428 429 // Returns true if the benchmark should continue through another iteration. 430 // NOTE: A benchmark may not return from the test until KeepRunning() has 431 // returned false. 432 bool KeepRunning() { 433 if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { 434 StartKeepRunning(); 435 } 436 bool const res = (--total_iterations_ != 0); 437 if (BENCHMARK_BUILTIN_EXPECT(!res, false)) { 438 FinishKeepRunning(); 439 } 440 return res; 441 } 442 443 // REQUIRES: timer is running and 'SkipWithError(...)' has not been called 444 // by the current thread. 445 // Stop the benchmark timer. If not called, the timer will be 446 // automatically stopped after the last iteration of the benchmark loop. 447 // 448 // For threaded benchmarks the PauseTiming() function only pauses the timing 449 // for the current thread. 450 // 451 // NOTE: The "real time" measurement is per-thread. If different threads 452 // report different measurements the largest one is reported. 453 // 454 // NOTE: PauseTiming()/ResumeTiming() are relatively 455 // heavyweight, and so their use should generally be avoided 456 // within each benchmark iteration, if possible. 457 void PauseTiming(); 458 459 // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called 460 // by the current thread. 461 // Start the benchmark timer. The timer is NOT running on entrance to the 462 // benchmark function. It begins running after control flow enters the 463 // benchmark loop. 464 // 465 // NOTE: PauseTiming()/ResumeTiming() are relatively 466 // heavyweight, and so their use should generally be avoided 467 // within each benchmark iteration, if possible. 468 void ResumeTiming(); 469 470 // REQUIRES: 'SkipWithError(...)' has not been called previously by the 471 // current thread. 472 // Report the benchmark as resulting in an error with the specified 'msg'. 473 // After this call the user may explicitly 'return' from the benchmark. 474 // 475 // If the ranged-for style of benchmark loop is used, the user must explicitly 476 // break from the loop, otherwise all future iterations will be run. 477 // If the 'KeepRunning()' loop is used the current thread will automatically 478 // exit the loop at the end of the current iteration. 479 // 480 // For threaded benchmarks only the current thread stops executing and future 481 // calls to `KeepRunning()` will block until all threads have completed 482 // the `KeepRunning()` loop. If multiple threads report an error only the 483 // first error message is used. 484 // 485 // NOTE: Calling 'SkipWithError(...)' does not cause the benchmark to exit 486 // the current scope immediately. If the function is called from within 487 // the 'KeepRunning()' loop the current iteration will finish. It is the users 488 // responsibility to exit the scope as needed. 489 void SkipWithError(const char* msg); 490 491 // REQUIRES: called exactly once per iteration of the benchmarking loop. 492 // Set the manually measured time for this benchmark iteration, which 493 // is used instead of automatically measured time if UseManualTime() was 494 // specified. 495 // 496 // For threaded benchmarks the final value will be set to the largest 497 // reported values. 498 void SetIterationTime(double seconds); 499 500 // Set the number of bytes processed by the current benchmark 501 // execution. This routine is typically called once at the end of a 502 // throughput oriented benchmark. If this routine is called with a 503 // value > 0, the report is printed in MB/sec instead of nanoseconds 504 // per iteration. 505 // 506 // REQUIRES: a benchmark has exited its benchmarking loop. 507 BENCHMARK_ALWAYS_INLINE 508 void SetBytesProcessed(size_t bytes) { bytes_processed_ = bytes; } 509 510 BENCHMARK_ALWAYS_INLINE 511 size_t bytes_processed() const { return bytes_processed_; } 512 513 // If this routine is called with complexity_n > 0 and complexity report is 514 // requested for the 515 // family benchmark, then current benchmark will be part of the computation 516 // and complexity_n will 517 // represent the length of N. 518 BENCHMARK_ALWAYS_INLINE 519 void SetComplexityN(int complexity_n) { complexity_n_ = complexity_n; } 520 521 BENCHMARK_ALWAYS_INLINE 522 int complexity_length_n() { return complexity_n_; } 523 524 // If this routine is called with items > 0, then an items/s 525 // label is printed on the benchmark report line for the currently 526 // executing benchmark. It is typically called at the end of a processing 527 // benchmark where a processing items/second output is desired. 528 // 529 // REQUIRES: a benchmark has exited its benchmarking loop. 530 BENCHMARK_ALWAYS_INLINE 531 void SetItemsProcessed(size_t items) { items_processed_ = items; } 532 533 BENCHMARK_ALWAYS_INLINE 534 size_t items_processed() const { return items_processed_; } 535 536 // If this routine is called, the specified label is printed at the 537 // end of the benchmark report line for the currently executing 538 // benchmark. Example: 539 // static void BM_Compress(benchmark::State& state) { 540 // ... 541 // double compress = input_size / output_size; 542 // state.SetLabel(StringPrintf("compress:%.1f%%", 100.0*compression)); 543 // } 544 // Produces output that looks like: 545 // BM_Compress 50 50 14115038 compress:27.3% 546 // 547 // REQUIRES: a benchmark has exited its benchmarking loop. 548 void SetLabel(const char* label); 549 550 void BENCHMARK_ALWAYS_INLINE SetLabel(const std::string& str) { 551 this->SetLabel(str.c_str()); 552 } 553 554 // Range arguments for this run. CHECKs if the argument has been set. 555 BENCHMARK_ALWAYS_INLINE 556 int range(std::size_t pos = 0) const { 557 assert(range_.size() > pos); 558 return range_[pos]; 559 } 560 561 BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead") 562 int range_x() const { return range(0); } 563 564 BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead") 565 int range_y() const { return range(1); } 566 567 BENCHMARK_ALWAYS_INLINE 568 size_t iterations() const { return (max_iterations - total_iterations_) + 1; } 569 570 private: 571 bool started_; 572 bool finished_; 573 size_t total_iterations_; 574 575 std::vector<int> range_; 576 577 size_t bytes_processed_; 578 size_t items_processed_; 579 580 int complexity_n_; 581 582 bool error_occurred_; 583 584 public: 585 // Container for user-defined counters. 586 UserCounters counters; 587 // Index of the executing thread. Values from [0, threads). 588 const int thread_index; 589 // Number of threads concurrently executing the benchmark. 590 const int threads; 591 const size_t max_iterations; 592 593 // TODO(EricWF) make me private 594 State(size_t max_iters, const std::vector<int>& ranges, int thread_i, 595 int n_threads, internal::ThreadTimer* timer, 596 internal::ThreadManager* manager); 597 598 private: 599 void StartKeepRunning(); 600 void FinishKeepRunning(); 601 internal::ThreadTimer* timer_; 602 internal::ThreadManager* manager_; 603 BENCHMARK_DISALLOW_COPY_AND_ASSIGN(State); 604 }; 605 606 struct State::StateIterator { 607 struct BENCHMARK_UNUSED Value {}; 608 typedef std::forward_iterator_tag iterator_category; 609 typedef Value value_type; 610 typedef Value reference; 611 typedef Value pointer; 612 613 private: 614 friend class State; 615 BENCHMARK_ALWAYS_INLINE 616 StateIterator() : cached_(0), parent_() {} 617 618 BENCHMARK_ALWAYS_INLINE 619 explicit StateIterator(State* st) 620 : cached_(st->error_occurred_ ? 0 : st->max_iterations), parent_(st) {} 621 622 public: 623 BENCHMARK_ALWAYS_INLINE 624 Value operator*() const { return Value(); } 625 626 BENCHMARK_ALWAYS_INLINE 627 StateIterator& operator++() { 628 assert(cached_ > 0); 629 --cached_; 630 return *this; 631 } 632 633 BENCHMARK_ALWAYS_INLINE 634 bool operator!=(StateIterator const&) const { 635 if (BENCHMARK_BUILTIN_EXPECT(cached_ != 0, true)) return true; 636 parent_->FinishKeepRunning(); 637 return false; 638 } 639 640 private: 641 size_t cached_; 642 State* const parent_; 643 }; 644 645 inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() { 646 return StateIterator(this); 647 } 648 inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() { 649 StartKeepRunning(); 650 return StateIterator(); 651 } 652 653 namespace internal { 654 655 typedef void(Function)(State&); 656 657 // ------------------------------------------------------ 658 // Benchmark registration object. The BENCHMARK() macro expands 659 // into an internal::Benchmark* object. Various methods can 660 // be called on this object to change the properties of the benchmark. 661 // Each method returns "this" so that multiple method calls can 662 // chained into one expression. 663 class Benchmark { 664 public: 665 virtual ~Benchmark(); 666 667 // Note: the following methods all return "this" so that multiple 668 // method calls can be chained together in one expression. 669 670 // Run this benchmark once with "x" as the extra argument passed 671 // to the function. 672 // REQUIRES: The function passed to the constructor must accept an arg1. 673 Benchmark* Arg(int x); 674 675 // Run this benchmark with the given time unit for the generated output report 676 Benchmark* Unit(TimeUnit unit); 677 678 // Run this benchmark once for a number of values picked from the 679 // range [start..limit]. (start and limit are always picked.) 680 // REQUIRES: The function passed to the constructor must accept an arg1. 681 Benchmark* Range(int start, int limit); 682 683 // Run this benchmark once for all values in the range [start..limit] with 684 // specific step 685 // REQUIRES: The function passed to the constructor must accept an arg1. 686 Benchmark* DenseRange(int start, int limit, int step = 1); 687 688 // Run this benchmark once with "args" as the extra arguments passed 689 // to the function. 690 // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... 691 Benchmark* Args(const std::vector<int>& args); 692 693 // Equivalent to Args({x, y}) 694 // NOTE: This is a legacy C++03 interface provided for compatibility only. 695 // New code should use 'Args'. 696 Benchmark* ArgPair(int x, int y) { 697 std::vector<int> args; 698 args.push_back(x); 699 args.push_back(y); 700 return Args(args); 701 } 702 703 // Run this benchmark once for a number of values picked from the 704 // ranges [start..limit]. (starts and limits are always picked.) 705 // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... 706 Benchmark* Ranges(const std::vector<std::pair<int, int> >& ranges); 707 708 // Equivalent to ArgNames({name}) 709 Benchmark* ArgName(const std::string& name); 710 711 // Set the argument names to display in the benchmark name. If not called, 712 // only argument values will be shown. 713 Benchmark* ArgNames(const std::vector<std::string>& names); 714 715 // Equivalent to Ranges({{lo1, hi1}, {lo2, hi2}}). 716 // NOTE: This is a legacy C++03 interface provided for compatibility only. 717 // New code should use 'Ranges'. 718 Benchmark* RangePair(int lo1, int hi1, int lo2, int hi2) { 719 std::vector<std::pair<int, int> > ranges; 720 ranges.push_back(std::make_pair(lo1, hi1)); 721 ranges.push_back(std::make_pair(lo2, hi2)); 722 return Ranges(ranges); 723 } 724 725 // Pass this benchmark object to *func, which can customize 726 // the benchmark by calling various methods like Arg, Args, 727 // Threads, etc. 728 Benchmark* Apply(void (*func)(Benchmark* benchmark)); 729 730 // Set the range multiplier for non-dense range. If not called, the range 731 // multiplier kRangeMultiplier will be used. 732 Benchmark* RangeMultiplier(int multiplier); 733 734 // Set the minimum amount of time to use when running this benchmark. This 735 // option overrides the `benchmark_min_time` flag. 736 // REQUIRES: `t > 0` and `Iterations` has not been called on this benchmark. 737 Benchmark* MinTime(double t); 738 739 // Specify the amount of iterations that should be run by this benchmark. 740 // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark. 741 // 742 // NOTE: This function should only be used when *exact* iteration control is 743 // needed and never to control or limit how long a benchmark runs, where 744 // `--benchmark_min_time=N` or `MinTime(...)` should be used instead. 745 Benchmark* Iterations(size_t n); 746 747 // Specify the amount of times to repeat this benchmark. This option overrides 748 // the `benchmark_repetitions` flag. 749 // REQUIRES: `n > 0` 750 Benchmark* Repetitions(int n); 751 752 // Specify if each repetition of the benchmark should be reported separately 753 // or if only the final statistics should be reported. If the benchmark 754 // is not repeated then the single result is always reported. 755 Benchmark* ReportAggregatesOnly(bool value = true); 756 757 // If a particular benchmark is I/O bound, runs multiple threads internally or 758 // if for some reason CPU timings are not representative, call this method. If 759 // called, the elapsed time will be used to control how many iterations are 760 // run, and in the printing of items/second or MB/seconds values. If not 761 // called, the cpu time used by the benchmark will be used. 762 Benchmark* UseRealTime(); 763 764 // If a benchmark must measure time manually (e.g. if GPU execution time is 765 // being 766 // measured), call this method. If called, each benchmark iteration should 767 // call 768 // SetIterationTime(seconds) to report the measured time, which will be used 769 // to control how many iterations are run, and in the printing of items/second 770 // or MB/second values. 771 Benchmark* UseManualTime(); 772 773 // Set the asymptotic computational complexity for the benchmark. If called 774 // the asymptotic computational complexity will be shown on the output. 775 Benchmark* Complexity(BigO complexity = benchmark::oAuto); 776 777 // Set the asymptotic computational complexity for the benchmark. If called 778 // the asymptotic computational complexity will be shown on the output. 779 Benchmark* Complexity(BigOFunc* complexity); 780 781 // Add this statistics to be computed over all the values of benchmark run 782 Benchmark* ComputeStatistics(std::string name, StatisticsFunc* statistics); 783 784 // Support for running multiple copies of the same benchmark concurrently 785 // in multiple threads. This may be useful when measuring the scaling 786 // of some piece of code. 787 788 // Run one instance of this benchmark concurrently in t threads. 789 Benchmark* Threads(int t); 790 791 // Pick a set of values T from [min_threads,max_threads]. 792 // min_threads and max_threads are always included in T. Run this 793 // benchmark once for each value in T. The benchmark run for a 794 // particular value t consists of t threads running the benchmark 795 // function concurrently. For example, consider: 796 // BENCHMARK(Foo)->ThreadRange(1,16); 797 // This will run the following benchmarks: 798 // Foo in 1 thread 799 // Foo in 2 threads 800 // Foo in 4 threads 801 // Foo in 8 threads 802 // Foo in 16 threads 803 Benchmark* ThreadRange(int min_threads, int max_threads); 804 805 // For each value n in the range, run this benchmark once using n threads. 806 // min_threads and max_threads are always included in the range. 807 // stride specifies the increment. E.g. DenseThreadRange(1, 8, 3) starts 808 // a benchmark with 1, 4, 7 and 8 threads. 809 Benchmark* DenseThreadRange(int min_threads, int max_threads, int stride = 1); 810 811 // Equivalent to ThreadRange(NumCPUs(), NumCPUs()) 812 Benchmark* ThreadPerCpu(); 813 814 virtual void Run(State& state) = 0; 815 816 // Used inside the benchmark implementation 817 struct Instance; 818 819 protected: 820 explicit Benchmark(const char* name); 821 Benchmark(Benchmark const&); 822 void SetName(const char* name); 823 824 int ArgsCnt() const; 825 826 static void AddRange(std::vector<int>* dst, int lo, int hi, int mult); 827 828 private: 829 friend class BenchmarkFamilies; 830 831 std::string name_; 832 ReportMode report_mode_; 833 std::vector<std::string> arg_names_; // Args for all benchmark runs 834 std::vector<std::vector<int> > args_; // Args for all benchmark runs 835 TimeUnit time_unit_; 836 int range_multiplier_; 837 double min_time_; 838 size_t iterations_; 839 int repetitions_; 840 bool use_real_time_; 841 bool use_manual_time_; 842 BigO complexity_; 843 BigOFunc* complexity_lambda_; 844 std::vector<Statistics> statistics_; 845 std::vector<int> thread_counts_; 846 847 Benchmark& operator=(Benchmark const&); 848 }; 849 850 } // namespace internal 851 852 // Create and register a benchmark with the specified 'name' that invokes 853 // the specified functor 'fn'. 854 // 855 // RETURNS: A pointer to the registered benchmark. 856 internal::Benchmark* RegisterBenchmark(const char* name, 857 internal::Function* fn); 858 859 #if defined(BENCHMARK_HAS_CXX11) 860 template <class Lambda> 861 internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); 862 #endif 863 864 // Remove all registered benchmarks. All pointers to previously registered 865 // benchmarks are invalidated. 866 void ClearRegisteredBenchmarks(); 867 868 namespace internal { 869 // The class used to hold all Benchmarks created from static function. 870 // (ie those created using the BENCHMARK(...) macros. 871 class FunctionBenchmark : public Benchmark { 872 public: 873 FunctionBenchmark(const char* name, Function* func) 874 : Benchmark(name), func_(func) {} 875 876 virtual void Run(State& st); 877 878 private: 879 Function* func_; 880 }; 881 882 #ifdef BENCHMARK_HAS_CXX11 883 template <class Lambda> 884 class LambdaBenchmark : public Benchmark { 885 public: 886 virtual void Run(State& st) { lambda_(st); } 887 888 private: 889 template <class OLambda> 890 LambdaBenchmark(const char* name, OLambda&& lam) 891 : Benchmark(name), lambda_(std::forward<OLambda>(lam)) {} 892 893 LambdaBenchmark(LambdaBenchmark const&) = delete; 894 895 private: 896 template <class Lam> 897 friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); 898 899 Lambda lambda_; 900 }; 901 #endif 902 903 } // namespace internal 904 905 inline internal::Benchmark* RegisterBenchmark(const char* name, 906 internal::Function* fn) { 907 return internal::RegisterBenchmarkInternal( 908 ::new internal::FunctionBenchmark(name, fn)); 909 } 910 911 #ifdef BENCHMARK_HAS_CXX11 912 template <class Lambda> 913 internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { 914 using BenchType = 915 internal::LambdaBenchmark<typename std::decay<Lambda>::type>; 916 return internal::RegisterBenchmarkInternal( 917 ::new BenchType(name, std::forward<Lambda>(fn))); 918 } 919 #endif 920 921 #if defined(BENCHMARK_HAS_CXX11) && \ 922 (!defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409) 923 template <class Lambda, class... Args> 924 internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, 925 Args&&... args) { 926 return benchmark::RegisterBenchmark( 927 name, [=](benchmark::State& st) { fn(st, args...); }); 928 } 929 #else 930 #define BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK 931 #endif 932 933 // The base class for all fixture tests. 934 class Fixture : public internal::Benchmark { 935 public: 936 Fixture() : internal::Benchmark("") {} 937 938 virtual void Run(State& st) { 939 this->SetUp(st); 940 this->BenchmarkCase(st); 941 this->TearDown(st); 942 } 943 944 // These will be deprecated ... 945 virtual void SetUp(const State&) {} 946 virtual void TearDown(const State&) {} 947 // ... In favor of these. 948 virtual void SetUp(State& st) { SetUp(const_cast<const State&>(st)); } 949 virtual void TearDown(State& st) { TearDown(const_cast<const State&>(st)); } 950 951 protected: 952 virtual void BenchmarkCase(State&) = 0; 953 }; 954 955 } // namespace benchmark 956 957 // ------------------------------------------------------ 958 // Macro to register benchmarks 959 960 // Check that __COUNTER__ is defined and that __COUNTER__ increases by 1 961 // every time it is expanded. X + 1 == X + 0 is used in case X is defined to be 962 // empty. If X is empty the expression becomes (+1 == +0). 963 #if defined(__COUNTER__) && (__COUNTER__ + 1 == __COUNTER__ + 0) 964 #define BENCHMARK_PRIVATE_UNIQUE_ID __COUNTER__ 965 #else 966 #define BENCHMARK_PRIVATE_UNIQUE_ID __LINE__ 967 #endif 968 969 // Helpers for generating unique variable names 970 #define BENCHMARK_PRIVATE_NAME(n) \ 971 BENCHMARK_PRIVATE_CONCAT(_benchmark_, BENCHMARK_PRIVATE_UNIQUE_ID, n) 972 #define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c) 973 #define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c 974 975 #define BENCHMARK_PRIVATE_DECLARE(n) \ 976 static ::benchmark::internal::Benchmark* BENCHMARK_PRIVATE_NAME(n) \ 977 BENCHMARK_UNUSED 978 979 #define BENCHMARK(n) \ 980 BENCHMARK_PRIVATE_DECLARE(n) = \ 981 (::benchmark::internal::RegisterBenchmarkInternal( \ 982 new ::benchmark::internal::FunctionBenchmark(#n, n))) 983 984 // Old-style macros 985 #define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) 986 #define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)}) 987 #define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t)) 988 #define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi)) 989 #define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ 990 BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}}) 991 992 #ifdef BENCHMARK_HAS_CXX11 993 994 // Register a benchmark which invokes the function specified by `func` 995 // with the additional arguments specified by `...`. 996 // 997 // For example: 998 // 999 // template <class ...ExtraArgs>` 1000 // void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { 1001 // [...] 1002 //} 1003 // /* Registers a benchmark named "BM_takes_args/int_string_test` */ 1004 // BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); 1005 #define BENCHMARK_CAPTURE(func, test_case_name, ...) \ 1006 BENCHMARK_PRIVATE_DECLARE(func) = \ 1007 (::benchmark::internal::RegisterBenchmarkInternal( \ 1008 new ::benchmark::internal::FunctionBenchmark( \ 1009 #func "/" #test_case_name, \ 1010 [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) 1011 1012 #endif // BENCHMARK_HAS_CXX11 1013 1014 // This will register a benchmark for a templatized function. For example: 1015 // 1016 // template<int arg> 1017 // void BM_Foo(int iters); 1018 // 1019 // BENCHMARK_TEMPLATE(BM_Foo, 1); 1020 // 1021 // will register BM_Foo<1> as a benchmark. 1022 #define BENCHMARK_TEMPLATE1(n, a) \ 1023 BENCHMARK_PRIVATE_DECLARE(n) = \ 1024 (::benchmark::internal::RegisterBenchmarkInternal( \ 1025 new ::benchmark::internal::FunctionBenchmark(#n "<" #a ">", n<a>))) 1026 1027 #define BENCHMARK_TEMPLATE2(n, a, b) \ 1028 BENCHMARK_PRIVATE_DECLARE(n) = \ 1029 (::benchmark::internal::RegisterBenchmarkInternal( \ 1030 new ::benchmark::internal::FunctionBenchmark(#n "<" #a "," #b ">", \ 1031 n<a, b>))) 1032 1033 #ifdef BENCHMARK_HAS_CXX11 1034 #define BENCHMARK_TEMPLATE(n, ...) \ 1035 BENCHMARK_PRIVATE_DECLARE(n) = \ 1036 (::benchmark::internal::RegisterBenchmarkInternal( \ 1037 new ::benchmark::internal::FunctionBenchmark( \ 1038 #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))) 1039 #else 1040 #define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) 1041 #endif 1042 1043 #define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ 1044 class BaseClass##_##Method##_Benchmark : public BaseClass { \ 1045 public: \ 1046 BaseClass##_##Method##_Benchmark() : BaseClass() { \ 1047 this->SetName(#BaseClass "/" #Method); \ 1048 } \ 1049 \ 1050 protected: \ 1051 virtual void BenchmarkCase(::benchmark::State&); \ 1052 }; 1053 1054 #define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ 1055 class BaseClass##_##Method##_Benchmark : public BaseClass<a> { \ 1056 public: \ 1057 BaseClass##_##Method##_Benchmark() : BaseClass<a>() { \ 1058 this->SetName(#BaseClass"<" #a ">/" #Method); \ 1059 } \ 1060 \ 1061 protected: \ 1062 virtual void BenchmarkCase(::benchmark::State&); \ 1063 }; 1064 1065 #define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ 1066 class BaseClass##_##Method##_Benchmark : public BaseClass<a, b> { \ 1067 public: \ 1068 BaseClass##_##Method##_Benchmark() : BaseClass<a, b>() { \ 1069 this->SetName(#BaseClass"<" #a "," #b ">/" #Method); \ 1070 } \ 1071 \ 1072 protected: \ 1073 virtual void BenchmarkCase(::benchmark::State&); \ 1074 }; 1075 1076 #ifdef BENCHMARK_HAS_CXX11 1077 #define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, ...) \ 1078 class BaseClass##_##Method##_Benchmark : public BaseClass<__VA_ARGS__> { \ 1079 public: \ 1080 BaseClass##_##Method##_Benchmark() : BaseClass<__VA_ARGS__>() { \ 1081 this->SetName(#BaseClass"<" #__VA_ARGS__ ">/" #Method); \ 1082 } \ 1083 \ 1084 protected: \ 1085 virtual void BenchmarkCase(::benchmark::State&); \ 1086 }; 1087 #else 1088 #define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(n, a) 1089 #endif 1090 1091 #define BENCHMARK_DEFINE_F(BaseClass, Method) \ 1092 BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ 1093 void BaseClass##_##Method##_Benchmark::BenchmarkCase 1094 1095 #define BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) \ 1096 BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ 1097 void BaseClass##_##Method##_Benchmark::BenchmarkCase 1098 1099 #define BENCHMARK_TEMPLATE2_DEFINE_F(BaseClass, Method, a, b) \ 1100 BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ 1101 void BaseClass##_##Method##_Benchmark::BenchmarkCase 1102 1103 #ifdef BENCHMARK_HAS_CXX11 1104 #define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, ...) \ 1105 BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ 1106 void BaseClass##_##Method##_Benchmark::BenchmarkCase 1107 #else 1108 #define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, a) BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) 1109 #endif 1110 1111 #define BENCHMARK_REGISTER_F(BaseClass, Method) \ 1112 BENCHMARK_PRIVATE_REGISTER_F(BaseClass##_##Method##_Benchmark) 1113 1114 #define BENCHMARK_PRIVATE_REGISTER_F(TestName) \ 1115 BENCHMARK_PRIVATE_DECLARE(TestName) = \ 1116 (::benchmark::internal::RegisterBenchmarkInternal(new TestName())) 1117 1118 // This macro will define and register a benchmark within a fixture class. 1119 #define BENCHMARK_F(BaseClass, Method) \ 1120 BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ 1121 BENCHMARK_REGISTER_F(BaseClass, Method); \ 1122 void BaseClass##_##Method##_Benchmark::BenchmarkCase 1123 1124 #define BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) \ 1125 BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ 1126 BENCHMARK_REGISTER_F(BaseClass, Method); \ 1127 void BaseClass##_##Method##_Benchmark::BenchmarkCase 1128 1129 #define BENCHMARK_TEMPLATE2_F(BaseClass, Method, a, b) \ 1130 BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ 1131 BENCHMARK_REGISTER_F(BaseClass, Method); \ 1132 void BaseClass##_##Method##_Benchmark::BenchmarkCase 1133 1134 #ifdef BENCHMARK_HAS_CXX11 1135 #define BENCHMARK_TEMPLATE_F(BaseClass, Method, ...) \ 1136 BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ 1137 BENCHMARK_REGISTER_F(BaseClass, Method); \ 1138 void BaseClass##_##Method##_Benchmark::BenchmarkCase 1139 #else 1140 #define BENCHMARK_TEMPLATE_F(BaseClass, Method, a) BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) 1141 #endif 1142 1143 // Helper macro to create a main routine in a test that runs the benchmarks 1144 #define BENCHMARK_MAIN() \ 1145 int main(int argc, char** argv) { \ 1146 ::benchmark::Initialize(&argc, argv); \ 1147 if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ 1148 ::benchmark::RunSpecifiedBenchmarks(); \ 1149 } \ 1150 int main(int, char**) 1151 1152 1153 // ------------------------------------------------------ 1154 // Benchmark Reporters 1155 1156 namespace benchmark { 1157 1158 struct CPUInfo { 1159 struct CacheInfo { 1160 std::string type; 1161 int level; 1162 int size; 1163 int num_sharing; 1164 }; 1165 1166 int num_cpus; 1167 double cycles_per_second; 1168 std::vector<CacheInfo> caches; 1169 bool scaling_enabled; 1170 1171 static const CPUInfo& Get(); 1172 1173 private: 1174 CPUInfo(); 1175 BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo); 1176 }; 1177 1178 // Interface for custom benchmark result printers. 1179 // By default, benchmark reports are printed to stdout. However an application 1180 // can control the destination of the reports by calling 1181 // RunSpecifiedBenchmarks and passing it a custom reporter object. 1182 // The reporter object must implement the following interface. 1183 class BenchmarkReporter { 1184 public: 1185 struct Context { 1186 CPUInfo const& cpu_info; 1187 // The number of chars in the longest benchmark name. 1188 size_t name_field_width; 1189 1190 Context(); 1191 }; 1192 1193 struct Run { 1194 Run() 1195 : error_occurred(false), 1196 iterations(1), 1197 time_unit(kNanosecond), 1198 real_accumulated_time(0), 1199 cpu_accumulated_time(0), 1200 bytes_per_second(0), 1201 items_per_second(0), 1202 max_heapbytes_used(0), 1203 complexity(oNone), 1204 complexity_lambda(), 1205 complexity_n(0), 1206 report_big_o(false), 1207 report_rms(false), 1208 counters() {} 1209 1210 std::string benchmark_name; 1211 std::string report_label; // Empty if not set by benchmark. 1212 bool error_occurred; 1213 std::string error_message; 1214 1215 int64_t iterations; 1216 TimeUnit time_unit; 1217 double real_accumulated_time; 1218 double cpu_accumulated_time; 1219 1220 // Return a value representing the real time per iteration in the unit 1221 // specified by 'time_unit'. 1222 // NOTE: If 'iterations' is zero the returned value represents the 1223 // accumulated time. 1224 double GetAdjustedRealTime() const; 1225 1226 // Return a value representing the cpu time per iteration in the unit 1227 // specified by 'time_unit'. 1228 // NOTE: If 'iterations' is zero the returned value represents the 1229 // accumulated time. 1230 double GetAdjustedCPUTime() const; 1231 1232 // Zero if not set by benchmark. 1233 double bytes_per_second; 1234 double items_per_second; 1235 1236 // This is set to 0.0 if memory tracing is not enabled. 1237 double max_heapbytes_used; 1238 1239 // Keep track of arguments to compute asymptotic complexity 1240 BigO complexity; 1241 BigOFunc* complexity_lambda; 1242 int complexity_n; 1243 1244 // what statistics to compute from the measurements 1245 const std::vector<Statistics>* statistics; 1246 1247 // Inform print function whether the current run is a complexity report 1248 bool report_big_o; 1249 bool report_rms; 1250 1251 UserCounters counters; 1252 }; 1253 1254 // Construct a BenchmarkReporter with the output stream set to 'std::cout' 1255 // and the error stream set to 'std::cerr' 1256 BenchmarkReporter(); 1257 1258 // Called once for every suite of benchmarks run. 1259 // The parameter "context" contains information that the 1260 // reporter may wish to use when generating its report, for example the 1261 // platform under which the benchmarks are running. The benchmark run is 1262 // never started if this function returns false, allowing the reporter 1263 // to skip runs based on the context information. 1264 virtual bool ReportContext(const Context& context) = 0; 1265 1266 // Called once for each group of benchmark runs, gives information about 1267 // cpu-time and heap memory usage during the benchmark run. If the group 1268 // of runs contained more than two entries then 'report' contains additional 1269 // elements representing the mean and standard deviation of those runs. 1270 // Additionally if this group of runs was the last in a family of benchmarks 1271 // 'reports' contains additional entries representing the asymptotic 1272 // complexity and RMS of that benchmark family. 1273 virtual void ReportRuns(const std::vector<Run>& report) = 0; 1274 1275 // Called once and only once after ever group of benchmarks is run and 1276 // reported. 1277 virtual void Finalize() {} 1278 1279 // REQUIRES: The object referenced by 'out' is valid for the lifetime 1280 // of the reporter. 1281 void SetOutputStream(std::ostream* out) { 1282 assert(out); 1283 output_stream_ = out; 1284 } 1285 1286 // REQUIRES: The object referenced by 'err' is valid for the lifetime 1287 // of the reporter. 1288 void SetErrorStream(std::ostream* err) { 1289 assert(err); 1290 error_stream_ = err; 1291 } 1292 1293 std::ostream& GetOutputStream() const { return *output_stream_; } 1294 1295 std::ostream& GetErrorStream() const { return *error_stream_; } 1296 1297 virtual ~BenchmarkReporter(); 1298 1299 // Write a human readable string to 'out' representing the specified 1300 // 'context'. 1301 // REQUIRES: 'out' is non-null. 1302 static void PrintBasicContext(std::ostream* out, Context const& context); 1303 1304 private: 1305 std::ostream* output_stream_; 1306 std::ostream* error_stream_; 1307 }; 1308 1309 // Simple reporter that outputs benchmark data to the console. This is the 1310 // default reporter used by RunSpecifiedBenchmarks(). 1311 class ConsoleReporter : public BenchmarkReporter { 1312 public: 1313 enum OutputOptions { 1314 OO_None = 0, 1315 OO_Color = 1, 1316 OO_Tabular = 2, 1317 OO_ColorTabular = OO_Color|OO_Tabular, 1318 OO_Defaults = OO_ColorTabular 1319 }; 1320 explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) 1321 : output_options_(opts_), name_field_width_(0), 1322 prev_counters_(), printed_header_(false) {} 1323 1324 virtual bool ReportContext(const Context& context); 1325 virtual void ReportRuns(const std::vector<Run>& reports); 1326 1327 protected: 1328 virtual void PrintRunData(const Run& report); 1329 virtual void PrintHeader(const Run& report); 1330 1331 OutputOptions output_options_; 1332 size_t name_field_width_; 1333 UserCounters prev_counters_; 1334 bool printed_header_; 1335 }; 1336 1337 class JSONReporter : public BenchmarkReporter { 1338 public: 1339 JSONReporter() : first_report_(true) {} 1340 virtual bool ReportContext(const Context& context); 1341 virtual void ReportRuns(const std::vector<Run>& reports); 1342 virtual void Finalize(); 1343 1344 private: 1345 void PrintRunData(const Run& report); 1346 1347 bool first_report_; 1348 }; 1349 1350 class CSVReporter : public BenchmarkReporter { 1351 public: 1352 CSVReporter() : printed_header_(false) {} 1353 virtual bool ReportContext(const Context& context); 1354 virtual void ReportRuns(const std::vector<Run>& reports); 1355 1356 private: 1357 void PrintRunData(const Run& report); 1358 1359 bool printed_header_; 1360 std::set< std::string > user_counter_names_; 1361 }; 1362 1363 inline const char* GetTimeUnitString(TimeUnit unit) { 1364 switch (unit) { 1365 case kMillisecond: 1366 return "ms"; 1367 case kMicrosecond: 1368 return "us"; 1369 case kNanosecond: 1370 default: 1371 return "ns"; 1372 } 1373 } 1374 1375 inline double GetTimeUnitMultiplier(TimeUnit unit) { 1376 switch (unit) { 1377 case kMillisecond: 1378 return 1e3; 1379 case kMicrosecond: 1380 return 1e6; 1381 case kNanosecond: 1382 default: 1383 return 1e9; 1384 } 1385 } 1386 1387 } // namespace benchmark 1388 1389 #endif // BENCHMARK_BENCHMARK_H_ 1390