1 // Copyright 2015 Google Inc. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 #ifndef BENCHMARK_REPORTER_H_ 15 #define BENCHMARK_REPORTER_H_ 16 17 #include <cassert> 18 #include <iosfwd> 19 #include <string> 20 #include <utility> 21 #include <vector> 22 #include <set> 23 24 #include "benchmark_api.h" // For forward declaration of BenchmarkReporter 25 26 namespace benchmark { 27 28 // Interface for custom benchmark result printers. 29 // By default, benchmark reports are printed to stdout. However an application 30 // can control the destination of the reports by calling 31 // RunSpecifiedBenchmarks and passing it a custom reporter object. 32 // The reporter object must implement the following interface. 33 class BenchmarkReporter { 34 public: 35 struct Context { 36 int num_cpus; 37 double mhz_per_cpu; 38 bool cpu_scaling_enabled; 39 40 // The number of chars in the longest benchmark name. 41 size_t name_field_width; 42 }; 43 44 struct Run { 45 Run() 46 : error_occurred(false), 47 iterations(1), 48 time_unit(kNanosecond), 49 real_accumulated_time(0), 50 cpu_accumulated_time(0), 51 bytes_per_second(0), 52 items_per_second(0), 53 max_heapbytes_used(0), 54 complexity(oNone), 55 complexity_lambda(), 56 complexity_n(0), 57 report_big_o(false), 58 report_rms(false), 59 counters() {} 60 61 std::string benchmark_name; 62 std::string report_label; // Empty if not set by benchmark. 63 bool error_occurred; 64 std::string error_message; 65 66 int64_t iterations; 67 TimeUnit time_unit; 68 double real_accumulated_time; 69 double cpu_accumulated_time; 70 71 // Return a value representing the real time per iteration in the unit 72 // specified by 'time_unit'. 73 // NOTE: If 'iterations' is zero the returned value represents the 74 // accumulated time. 75 double GetAdjustedRealTime() const; 76 77 // Return a value representing the cpu time per iteration in the unit 78 // specified by 'time_unit'. 79 // NOTE: If 'iterations' is zero the returned value represents the 80 // accumulated time. 81 double GetAdjustedCPUTime() const; 82 83 // Zero if not set by benchmark. 84 double bytes_per_second; 85 double items_per_second; 86 87 // This is set to 0.0 if memory tracing is not enabled. 88 double max_heapbytes_used; 89 90 // Keep track of arguments to compute asymptotic complexity 91 BigO complexity; 92 BigOFunc* complexity_lambda; 93 int complexity_n; 94 95 // Inform print function whether the current run is a complexity report 96 bool report_big_o; 97 bool report_rms; 98 99 UserCounters counters; 100 }; 101 102 // Construct a BenchmarkReporter with the output stream set to 'std::cout' 103 // and the error stream set to 'std::cerr' 104 BenchmarkReporter(); 105 106 // Called once for every suite of benchmarks run. 107 // The parameter "context" contains information that the 108 // reporter may wish to use when generating its report, for example the 109 // platform under which the benchmarks are running. The benchmark run is 110 // never started if this function returns false, allowing the reporter 111 // to skip runs based on the context information. 112 virtual bool ReportContext(const Context& context) = 0; 113 114 // Called once for each group of benchmark runs, gives information about 115 // cpu-time and heap memory usage during the benchmark run. If the group 116 // of runs contained more than two entries then 'report' contains additional 117 // elements representing the mean and standard deviation of those runs. 118 // Additionally if this group of runs was the last in a family of benchmarks 119 // 'reports' contains additional entries representing the asymptotic 120 // complexity and RMS of that benchmark family. 121 virtual void ReportRuns(const std::vector<Run>& report) = 0; 122 123 // Called once and only once after ever group of benchmarks is run and 124 // reported. 125 virtual void Finalize() {} 126 127 // REQUIRES: The object referenced by 'out' is valid for the lifetime 128 // of the reporter. 129 void SetOutputStream(std::ostream* out) { 130 assert(out); 131 output_stream_ = out; 132 } 133 134 // REQUIRES: The object referenced by 'err' is valid for the lifetime 135 // of the reporter. 136 void SetErrorStream(std::ostream* err) { 137 assert(err); 138 error_stream_ = err; 139 } 140 141 std::ostream& GetOutputStream() const { return *output_stream_; } 142 143 std::ostream& GetErrorStream() const { return *error_stream_; } 144 145 virtual ~BenchmarkReporter(); 146 147 // Write a human readable string to 'out' representing the specified 148 // 'context'. 149 // REQUIRES: 'out' is non-null. 150 static void PrintBasicContext(std::ostream* out, Context const& context); 151 152 private: 153 std::ostream* output_stream_; 154 std::ostream* error_stream_; 155 }; 156 157 // Simple reporter that outputs benchmark data to the console. This is the 158 // default reporter used by RunSpecifiedBenchmarks(). 159 class ConsoleReporter : public BenchmarkReporter { 160 public: 161 enum OutputOptions { OO_None, OO_Color }; 162 explicit ConsoleReporter(OutputOptions color_output = OO_Color) 163 : name_field_width_(0), color_output_(color_output == OO_Color) {} 164 165 virtual bool ReportContext(const Context& context); 166 virtual void ReportRuns(const std::vector<Run>& reports); 167 168 protected: 169 virtual void PrintRunData(const Run& report); 170 virtual void PrintHeader(const Run& report); 171 172 size_t name_field_width_; 173 bool printed_header_; 174 175 private: 176 bool color_output_; 177 }; 178 179 class JSONReporter : public BenchmarkReporter { 180 public: 181 JSONReporter() : first_report_(true) {} 182 virtual bool ReportContext(const Context& context); 183 virtual void ReportRuns(const std::vector<Run>& reports); 184 virtual void Finalize(); 185 186 private: 187 void PrintRunData(const Run& report); 188 189 bool first_report_; 190 }; 191 192 class CSVReporter : public BenchmarkReporter { 193 public: 194 CSVReporter() : printed_header_(false) {} 195 virtual bool ReportContext(const Context& context); 196 virtual void ReportRuns(const std::vector<Run>& reports); 197 198 private: 199 void PrintRunData(const Run& report); 200 201 bool printed_header_; 202 std::set< std::string > user_counter_names_; 203 }; 204 205 inline const char* GetTimeUnitString(TimeUnit unit) { 206 switch (unit) { 207 case kMillisecond: 208 return "ms"; 209 case kMicrosecond: 210 return "us"; 211 case kNanosecond: 212 default: 213 return "ns"; 214 } 215 } 216 217 inline double GetTimeUnitMultiplier(TimeUnit unit) { 218 switch (unit) { 219 case kMillisecond: 220 return 1e3; 221 case kMicrosecond: 222 return 1e6; 223 case kNanosecond: 224 default: 225 return 1e9; 226 } 227 } 228 229 } // end namespace benchmark 230 #endif // BENCHMARK_REPORTER_H_ 231
