1 #ifndef TEST_OUTPUT_TEST_H 2 #define TEST_OUTPUT_TEST_H 3 4 #undef NDEBUG 5 #include <initializer_list> 6 #include <memory> 7 #include <string> 8 #include <utility> 9 #include <vector> 10 #include <functional> 11 #include <sstream> 12 13 #include "../src/re.h" 14 #include "benchmark/benchmark.h" 15 16 #define CONCAT2(x, y) x##y 17 #define CONCAT(x, y) CONCAT2(x, y) 18 19 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__) 20 21 #define SET_SUBSTITUTIONS(...) \ 22 int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__) 23 24 enum MatchRules { 25 MR_Default, // Skip non-matching lines until a match is found. 26 MR_Next, // Match must occur on the next line. 27 MR_Not // No line between the current position and the next match matches 28 // the regex 29 }; 30 31 struct TestCase { 32 TestCase(std::string re, int rule = MR_Default); 33 34 std::string regex_str; 35 int match_rule; 36 std::string substituted_regex; 37 std::shared_ptr<benchmark::Regex> regex; 38 }; 39 40 enum TestCaseID { 41 TC_ConsoleOut, 42 TC_ConsoleErr, 43 TC_JSONOut, 44 TC_JSONErr, 45 TC_CSVOut, 46 TC_CSVErr, 47 48 TC_NumID // PRIVATE 49 }; 50 51 // Add a list of test cases to be run against the output specified by 52 // 'ID' 53 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il); 54 55 // Add or set a list of substitutions to be performed on constructed regex's 56 // See 'output_test_helper.cc' for a list of default substitutions. 57 int SetSubstitutions( 58 std::initializer_list<std::pair<std::string, std::string>> il); 59 60 // Run all output tests. 61 void RunOutputTests(int argc, char* argv[]); 62 63 // ========================================================================= // 64 // ------------------------- Results checking ------------------------------ // 65 // ========================================================================= // 66 67 // Call this macro to register a benchmark for checking its results. This 68 // should be all that's needed. It subscribes a function to check the (CSV) 69 // results of a benchmark. This is done only after verifying that the output 70 // strings are really as expected. 71 // bm_name_pattern: a name or a regex pattern which will be matched against 72 // all the benchmark names. Matching benchmarks 73 // will be the subject of a call to checker_function 74 // checker_function: should be of type ResultsCheckFn (see below) 75 #define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \ 76 size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function) 77 78 struct Results; 79 typedef std::function< void(Results const&) > ResultsCheckFn; 80 81 size_t AddChecker(const char* bm_name_pattern, ResultsCheckFn fn); 82 83 // Class holding the results of a benchmark. 84 // It is passed in calls to checker functions. 85 struct Results { 86 87 // the benchmark name 88 std::string name; 89 // the benchmark fields 90 std::map< std::string, std::string > values; 91 92 Results(const std::string& n) : name(n) {} 93 94 int NumThreads() const; 95 96 typedef enum { kCpuTime, kRealTime } BenchmarkTime; 97 98 // get cpu_time or real_time in seconds 99 double GetTime(BenchmarkTime which) const; 100 101 // get the real_time duration of the benchmark in seconds. 102 // it is better to use fuzzy float checks for this, as the float 103 // ASCII formatting is lossy. 104 double DurationRealTime() const { 105 return GetAs< double >("iterations") * GetTime(kRealTime); 106 } 107 // get the cpu_time duration of the benchmark in seconds 108 double DurationCPUTime() const { 109 return GetAs< double >("iterations") * GetTime(kCpuTime); 110 } 111 112 // get the string for a result by name, or nullptr if the name 113 // is not found 114 const std::string* Get(const char* entry_name) const { 115 auto it = values.find(entry_name); 116 if(it == values.end()) return nullptr; 117 return &it->second; 118 } 119 120 // get a result by name, parsed as a specific type. 121 // NOTE: for counters, use GetCounterAs instead. 122 template <class T> 123 T GetAs(const char* entry_name) const; 124 125 // counters are written as doubles, so they have to be read first 126 // as a double, and only then converted to the asked type. 127 template <class T> 128 T GetCounterAs(const char* entry_name) const { 129 double dval = GetAs< double >(entry_name); 130 T tval = static_cast< T >(dval); 131 return tval; 132 } 133 }; 134 135 template <class T> 136 T Results::GetAs(const char* entry_name) const { 137 auto *sv = Get(entry_name); 138 CHECK(sv != nullptr && !sv->empty()); 139 std::stringstream ss; 140 ss << *sv; 141 T out; 142 ss >> out; 143 CHECK(!ss.fail()); 144 return out; 145 } 146 147 //---------------------------------- 148 // Macros to help in result checking. Do not use them with arguments causing 149 // side-effects. 150 151 #define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \ 152 CONCAT(CHECK_, relationship) \ 153 (entry.getfn< var_type >(var_name), (value)) << "\n" \ 154 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ 155 << __FILE__ << ":" << __LINE__ << ": " \ 156 << "expected (" << #var_type << ")" << (var_name) \ 157 << "=" << (entry).getfn< var_type >(var_name) \ 158 << " to be " #relationship " to " << (value) << "\n" 159 160 // check with tolerance. eps_factor is the tolerance window, which is 161 // interpreted relative to value (eg, 0.1 means 10% of value). 162 #define _CHECK_FLOAT_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value, eps_factor) \ 163 CONCAT(CHECK_FLOAT_, relationship) \ 164 (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \ 165 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ 166 << __FILE__ << ":" << __LINE__ << ": " \ 167 << "expected (" << #var_type << ")" << (var_name) \ 168 << "=" << (entry).getfn< var_type >(var_name) \ 169 << " to be " #relationship " to " << (value) << "\n" \ 170 << __FILE__ << ":" << __LINE__ << ": " \ 171 << "with tolerance of " << (eps_factor) * (value) \ 172 << " (" << (eps_factor)*100. << "%), " \ 173 << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \ 174 << " (" << (((entry).getfn< var_type >(var_name) - (value)) \ 175 / \ 176 ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \ 177 << "%)" 178 179 #define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \ 180 _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value) 181 182 #define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \ 183 _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value) 184 185 #define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \ 186 _CHECK_FLOAT_RESULT_VALUE(entry, GetAs, double, var_name, relationship, value, eps_factor) 187 188 #define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \ 189 _CHECK_FLOAT_RESULT_VALUE(entry, GetCounterAs, double, var_name, relationship, value, eps_factor) 190 191 // ========================================================================= // 192 // --------------------------- Misc Utilities ------------------------------ // 193 // ========================================================================= // 194 195 namespace { 196 197 const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; 198 199 } // end namespace 200 201 #endif // TEST_OUTPUT_TEST_H 202
