1 The lldb-perf infrastructure for LLDB performance testing 2 =========================================================== 3 4 lldb-perf is an infrastructure meant to simplify the creation of performance 5 tests for the LLDB debugger. It is contained in liblldbperf.a which is part of 6 the standard opensource checkout of LLDB 7 8 Its main concepts are: 9 - Gauges: a gauge is a thing that takes a sample. Samples include elapsed time, 10 memory used, and energy consumed. 11 - Metrics: a metric is a collection of samples that knows how to do statistics 12 like sum() and average(). Metrics can be extended as needed. 13 - Measurements: a measurement is the thing that stores an action, a gauge and 14 a metric. You define measurements as in take the time to run this function, 15 take the memory to run this block of code, and then after you invoke it, 16 your stats will automagically be there. 17 - Tests: a test is a sequence of steps and measurements. 18 19 Tests cases should be added as targets to the lldbperf.xcodeproj project. It 20 is probably easiest to duplicate one of the existing targets. In order to 21 write a test based on lldb-perf, you need to subclass lldb_perf::TestCase: 22 23 using namespace lldb_perf; 24 25 class FormattersTest : public TestCase 26 { 27 28 Usually, you will define measurements as variables of your test case class: 29 30 private: 31 // C++ formatters 32 TimeMeasurement<std::function<void(SBValue)>> m_dump_std_vector_measurement; 33 TimeMeasurement<std::function<void(SBValue)>> m_dump_std_list_measurement; 34 TimeMeasurement<std::function<void(SBValue)>> m_dump_std_map_measurement; 35 TimeMeasurement<std::function<void(SBValue)>> m_dump_std_string_measurement; 36 37 // Cocoa formatters 38 TimeMeasurement<std::function<void(SBValue)>> m_dump_nsstring_measurement; 39 TimeMeasurement<std::function<void(SBValue)>> m_dump_nsarray_measurement; 40 TimeMeasurement<std::function<void(SBValue)>> m_dump_nsdictionary_measurement; 41 TimeMeasurement<std::function<void(SBValue)>> m_dump_nsset_measurement; 42 TimeMeasurement<std::function<void(SBValue)>> m_dump_nsbundle_measurement; 43 TimeMeasurement<std::function<void(SBValue)>> m_dump_nsdate_measurement; 44 45 A TimeMeasurement is, obviously, a class that measures how much time to run 46 this block of code. The block of code is passed as an std::function which you 47 can construct with a lambda! You need to give the prototype of your block of 48 code. In this example, we run blocks of code that take an SBValue and return 49 nothing. 50 51 These blocks look like: 52 53 m_dump_std_vector_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 54 lldb_perf::Xcode::FetchVariable (value,1,false); 55 }, "std-vector", "time to dump an std::vector"); 56 57 Here we are saying: make me a measurement named std-vector, whose 58 description is time to dump an std::vector and that takes the time required 59 to call lldb_perf::Xcode::FetchVariable(value,1,false). 60 61 The Xcode class is a collection of utility functions that replicate common 62 Xcode patterns (FetchVariable unsurprisingly calls API functions that Xcode 63 could use when populating a variables view entry - the 1 means expand 1 level 64 of depth and the false means do not dump the data to stdout) 65 66 A full constructor for a TestCase looks like: 67 68 FormattersTest () : TestCase() 69 { 70 m_dump_std_vector_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 71 lldb_perf::Xcode::FetchVariable (value,1,false); 72 }, "std-vector", "time to dump an std::vector"); 73 m_dump_std_list_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 74 lldb_perf::Xcode::FetchVariable (value,1,false); 75 }, "std-list", "time to dump an std::list"); 76 m_dump_std_map_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 77 lldb_perf::Xcode::FetchVariable (value,1,false); 78 }, "std-map", "time to dump an std::map"); 79 m_dump_std_string_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 80 lldb_perf::Xcode::FetchVariable (value,1,false); 81 }, "std-string", "time to dump an std::string"); 82 83 m_dump_nsstring_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 84 lldb_perf::Xcode::FetchVariable (value,0,false); 85 }, "ns-string", "time to dump an NSString"); 86 87 m_dump_nsarray_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 88 lldb_perf::Xcode::FetchVariable (value,1,false); 89 }, "ns-array", "time to dump an NSArray"); 90 91 m_dump_nsdictionary_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 92 lldb_perf::Xcode::FetchVariable (value,1,false); 93 }, "ns-dictionary", "time to dump an NSDictionary"); 94 95 m_dump_nsset_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 96 lldb_perf::Xcode::FetchVariable (value,1,false); 97 }, "ns-set", "time to dump an NSSet"); 98 99 m_dump_nsbundle_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 100 lldb_perf::Xcode::FetchVariable (value,1,false); 101 }, "ns-bundle", "time to dump an NSBundle"); 102 103 m_dump_nsdate_measurement = CreateTimeMeasurement([] (SBValue value) -> void { 104 lldb_perf::Xcode::FetchVariable (value,0,false); 105 }, "ns-date", "time to dump an NSDate"); 106 } 107 108 Once your test case is constructed, Setup() is called on it: 109 110 virtual bool 111 Setup (int argc, const char** argv) 112 { 113 m_app_path.assign(argv[1]); 114 m_out_path.assign(argv[2]); 115 m_target = m_debugger.CreateTarget(m_app_path.c_str()); 116 m_target.BreakpointCreateByName("main"); 117 SBLaunchInfo launch_info (argv); 118 return Launch (launch_info); 119 } 120 121 Setup() returns a boolean value that indicates if setup was successful. 122 In Setup() you fill out a SBLaunchInfo with any needed settings for launching 123 your process like arguments, environment variables, working directory, and 124 much more. 125 126 The last thing you want to do in setup is call Launch(): 127 128 bool 129 Launch (coSBLaunchInfo &launch_info); 130 131 This ensures your target is now alive. Make sure to have a breakpoint created. 132 133 Once you launched, the event loop is entered. The event loop waits for stops, 134 and when it gets one, it calls your test cases TestStep() function: 135 136 virtual void 137 TestStep (int counter, ActionWanted &next_action) 138 139 the counter is the step id (a monotonically increasing counter). In TestStep() 140 you will essentially run your measurements and then return what you want the 141 driver to do by filling in the ActionWanted object named "next_action". 142 143 Possible options are: 144 - continue process next_action.Continue(); 145 - kill process next_action.Kill(); 146 - Step-out on a thread next_action.StepOut(SBThread) 147 - step-over on a thread. next_action.StepOver(SBThread) 148 149 If you use ActionWanted::Next() or ActionWanted::Finish() you need to specify 150 a thread to use. By default the TestCase class will select the first thread 151 that had a stop reason other than eStopReasonNone and place it into the 152 m_thread member variable of TestCase. This means if your test case hits a 153 breakpoint or steps, the thread that hit the breakpoint or finished the step 154 will automatically be selected in the process (m_process) and m_thread will 155 be set to this thread. If you have one or more threads that will stop with a 156 reason simultaneously, you will need to find those threads manually by 157 iterating through the process list and determine what to do next. 158 159 For your convenience TestCase has m_debugger, m_target and m_process as member 160 variables. As state above m_thread will be filled in with the first thread 161 that has a stop reason. 162 163 An example: 164 165 virtual void 166 TestStep (int counter, ActionWanted &next_action) 167 { 168 case 0: 169 m_target.BreakpointCreateByLocation("fmts_tester.mm", 68); 170 next_action.Continue(); 171 break; 172 case 1: 173 DoTest (); 174 next_action.Continue(); 175 break; 176 case 2: 177 DoTest (); 178 next_action.StepOver(m_thread); 179 break; 180 181 DoTest() is a function I define in my own class that calls the measurements: 182 void 183 DoTest () 184 { 185 SBThread thread_main(m_thread); 186 SBFrame frame_zero(thread_main.GetFrameAtIndex(0)); 187 188 m_dump_nsarray_measurement(frame_zero.FindVariable("nsarray", lldb::eDynamicCanRunTarget)); 189 m_dump_nsarray_measurement(frame_zero.FindVariable("nsmutablearray", lldb::eDynamicCanRunTarget)); 190 191 m_dump_nsdictionary_measurement(frame_zero.FindVariable("nsdictionary", lldb::eDynamicCanRunTarget)); 192 m_dump_nsdictionary_measurement(frame_zero.FindVariable("nsmutabledictionary", lldb::eDynamicCanRunTarget)); 193 194 m_dump_nsstring_measurement(frame_zero.FindVariable("str0", lldb::eDynamicCanRunTarget)); 195 m_dump_nsstring_measurement(frame_zero.FindVariable("str1", lldb::eDynamicCanRunTarget)); 196 m_dump_nsstring_measurement(frame_zero.FindVariable("str2", lldb::eDynamicCanRunTarget)); 197 m_dump_nsstring_measurement(frame_zero.FindVariable("str3", lldb::eDynamicCanRunTarget)); 198 m_dump_nsstring_measurement(frame_zero.FindVariable("str4", lldb::eDynamicCanRunTarget)); 199 200 m_dump_nsdate_measurement(frame_zero.FindVariable("me", lldb::eDynamicCanRunTarget)); 201 m_dump_nsdate_measurement(frame_zero.FindVariable("cutie", lldb::eDynamicCanRunTarget)); 202 m_dump_nsdate_measurement(frame_zero.FindVariable("mom", lldb::eDynamicCanRunTarget)); 203 m_dump_nsdate_measurement(frame_zero.FindVariable("dad", lldb::eDynamicCanRunTarget)); 204 m_dump_nsdate_measurement(frame_zero.FindVariable("today", lldb::eDynamicCanRunTarget)); 205 206 m_dump_nsbundle_measurement(frame_zero.FindVariable("bundles", lldb::eDynamicCanRunTarget)); 207 m_dump_nsbundle_measurement(frame_zero.FindVariable("frameworks", lldb::eDynamicCanRunTarget)); 208 209 m_dump_nsset_measurement(frame_zero.FindVariable("nsset", lldb::eDynamicCanRunTarget)); 210 m_dump_nsset_measurement(frame_zero.FindVariable("nsmutableset", lldb::eDynamicCanRunTarget)); 211 212 m_dump_std_vector_measurement(frame_zero.FindVariable("vector", lldb::eDynamicCanRunTarget)); 213 m_dump_std_list_measurement(frame_zero.FindVariable("list", lldb::eDynamicCanRunTarget)); 214 m_dump_std_map_measurement(frame_zero.FindVariable("map", lldb::eDynamicCanRunTarget)); 215 216 m_dump_std_string_measurement(frame_zero.FindVariable("sstr0", lldb::eDynamicCanRunTarget)); 217 m_dump_std_string_measurement(frame_zero.FindVariable("sstr1", lldb::eDynamicCanRunTarget)); 218 m_dump_std_string_measurement(frame_zero.FindVariable("sstr2", lldb::eDynamicCanRunTarget)); 219 m_dump_std_string_measurement(frame_zero.FindVariable("sstr3", lldb::eDynamicCanRunTarget)); 220 m_dump_std_string_measurement(frame_zero.FindVariable("sstr4", lldb::eDynamicCanRunTarget)); 221 } 222 223 Essentially, you call your measurements as if they were functions, passing 224 them arguments and all, and they will do the right thing with gathering stats. 225 226 The last step is usually to KILL the inferior and bail out: 227 228 virtual ActionWanted 229 TestStep (int counter) 230 { 231 ... 232 case 9: 233 DoTest (); 234 next_action.Continue(); 235 break; 236 case 10: 237 DoTest (); 238 next_action.Continue(); 239 break; 240 default: 241 next_action.Kill(); 242 break; 243 } 244 245 246 At the end, you define a Results() function: 247 248 void 249 Results () 250 { 251 CFCMutableArray array; 252 m_dump_std_vector_measurement.Write(array); 253 m_dump_std_list_measurement.Write(array); 254 m_dump_std_map_measurement.Write(array); 255 m_dump_std_string_measurement.Write(array); 256 257 m_dump_nsstring_measurement.Write(array); 258 m_dump_nsarray_measurement.Write(array); 259 m_dump_nsdictionary_measurement.Write(array); 260 m_dump_nsset_measurement.Write(array); 261 m_dump_nsbundle_measurement.Write(array); 262 m_dump_nsdate_measurement.Write(array); 263 264 CFDataRef xmlData = CFPropertyListCreateData (kCFAllocatorDefault, 265 array.get(), 266 kCFPropertyListXMLFormat_v1_0, 267 0, 268 NULL); 269 270 CFURLRef file = CFURLCreateFromFileSystemRepresentation (NULL, 271 (const UInt8*)m_out_path.c_str(), 272 m_out_path.size(), 273 FALSE); 274 275 CFURLWriteDataAndPropertiesToResource(file,xmlData,NULL,NULL); 276 } 277 278 For now, pretty much copy this and just call Write() on all your measurements. 279 I plan to move this higher in the hierarchy (e.g. make a 280 TestCase::Write(filename) fairly soon). 281 282 Your main() will look like: 283 284 int main(int argc, const char * argv[]) 285 { 286 MyTest test; 287 TestCase::Run (test, argc, argv); 288 return 0; 289 } 290 291 If you are debugging your test, before Run() call 292 293 test.SetVerbose(true); 294 295 Feel free to send any questions and ideas for improvements. 296
