1 #!/usr/bin/env ruby 2 3 # Copyright (C) 2011 Apple Inc. All rights reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions 7 # are met: 8 # 1. Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # 2. Redistributions in binary form must reproduce the above copyright 11 # notice, this list of conditions and the following disclaimer in the 12 # documentation and/or other materials provided with the distribution. 13 # 14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' 15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS 18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 24 # THE POSSIBILITY OF SUCH DAMAGE. 25 26 require 'rubygems' 27 28 require 'getoptlong' 29 require 'pathname' 30 require 'tempfile' 31 require 'socket' 32 33 begin 34 require 'json' 35 rescue LoadError => e 36 $stderr.puts "It does not appear that you have the 'json' package installed. Try running 'sudo gem install json'." 37 exit 1 38 end 39 40 # Configuration 41 42 CONFIGURATION_FLNM = ENV["HOME"]+"/.bencher" 43 44 unless FileTest.exist? CONFIGURATION_FLNM 45 $stderr.puts "Error: no configuration file at ~/.bencher." 46 $stderr.puts "This file should contain paths to SunSpider, V8, and Kraken, as well as a" 47 $stderr.puts "temporary directory that bencher can use for its remote mode. It should be" 48 $stderr.puts "formatted in JSON. For example:" 49 $stderr.puts "{" 50 $stderr.puts " \"sunSpiderPath\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/sunspider-1.0\"," 51 $stderr.puts " \"v8Path\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/v8-v6\"," 52 $stderr.puts " \"krakenPath\": \"/Volumes/Data/pizlo/kraken/kraken-e119421cb325/tests/kraken-1.1\"," 53 $stderr.puts " \"tempPath\": \"/Volumes/Data/pizlo/bencher/temp\"" 54 $stderr.puts "}" 55 exit 1 56 end 57 58 CONFIGURATION = JSON.parse(File::read(CONFIGURATION_FLNM)) 59 60 SUNSPIDER_PATH = CONFIGURATION["sunSpiderPath"] 61 V8_PATH = CONFIGURATION["v8Path"] 62 KRAKEN_PATH = CONFIGURATION["krakenPath"] 63 TEMP_PATH = CONFIGURATION["tempPath"] 64 BENCH_DATA_PATH = TEMP_PATH + "/benchdata" 65 66 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 67 0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 68 0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 69 0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 70 0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 71 0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 72 0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 73 0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 74 0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 75 0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 76 0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 77 0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 78 0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 79 0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 80 0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 81 0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 82 0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 83 0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 84 0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 85 0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 86 0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 87 0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 88 0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 89 0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 90 0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 91 0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 92 0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 93 0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 94 0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 95 0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 96 0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 97 0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 98 0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 99 0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 100 0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 101 0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 102 0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 103 0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 104 0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 105 0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 106 0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 107 0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 108 0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 109 0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 110 0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 111 0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 112 0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 113 0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 114 0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 115 0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 116 0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 117 0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 118 0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 119 0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 120 0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 121 0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 122 0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 123 0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 124 0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 125 0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 126 0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 127 0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 128 0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 129 0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 130 0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 131 0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 132 0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 133 0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 134 0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 135 0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 136 0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 137 0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 138 0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 139 0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 140 0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 141 0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 142 0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 143 0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 144 0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 145 0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 146 0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 147 0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 148 0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 149 0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 150 0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 151 0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 152 0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 153 0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 154 0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 155 0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 156 0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 157 0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 158 0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 159 0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 160 0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 161 0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 162 0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 163 0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 164 0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 165 0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 166 0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 167 0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 168 0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 169 0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 170 0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 171 0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 172 0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 173 0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 174 0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 175 0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 176 0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 177 0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 178 0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 179 0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 180 0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 181 0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 182 0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 183 0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 184 0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 185 0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 186 0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 187 0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 188 0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 189 0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 190 0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 191 0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 192 0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 193 0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 194 0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 195 0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 196 0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 197 0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 198 0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 199 0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 200 0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 201 0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 202 0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 203 0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 204 0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 205 0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 206 0.99616, 0.996164] 207 208 # Run-time configuration parameters (can be set with command-line options) 209 210 $rerun=1 211 $inner=3 212 $warmup=1 213 $outer=4 214 $includeSunSpider=true 215 $includeV8=true 216 $includeKraken=true 217 $measureGC=false 218 $benchmarkPattern=nil 219 $verbosity=0 220 $timeMode=:preciseTime 221 $forceVMKind=nil 222 $brief=false 223 $silent=false 224 $remoteHosts=[] 225 $alsoLocal=false 226 $sshOptions=[] 227 $vms = [] 228 $needToCopyVMs = false 229 $dontCopyVMs = false 230 231 $prepare = true 232 $run = true 233 $analyze = [] 234 235 # Helpful functions and classes 236 237 def smallUsage 238 puts "Use the --help option to get basic usage information." 239 exit 1 240 end 241 242 def usage 243 puts "bencher [options] <vm1> [<vm2> ...]" 244 puts 245 puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken" 246 puts "benchmarks, and reports detailed statistics. What makes bencher special is" 247 puts "that each benchmark/VM configuration is run in a single VM invocation, and" 248 puts "the invocations are run in random order. This minimizes systematics due to" 249 puts "one benchmark polluting the running time of another. The fine-grained" 250 puts "interleaving of VM invocations further minimizes systematics due to changes in" 251 puts "the performance or behavior of your machine." 252 puts 253 puts "Bencher is highly configurable. You can compare as many VMs as you like. You" 254 puts "can change the amount of warm-up iterations, number of iterations executed per" 255 puts "VM invocation, and the number of VM invocations per benchmark. By default," 256 puts "SunSpider, VM, and Kraken are all run; but you can run any combination of these" 257 puts "suites." 258 puts 259 puts "The <vm> should be either a path to a JavaScript runtime executable (such as" 260 puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to" 261 puts "the executable and <name> is the name that you would like to give the" 262 puts "configuration for the purposeof reporting. If no name is given, a generic name" 263 puts "of the form Conf#<n> will be ascribed to the configuration automatically." 264 puts 265 puts "Options:" 266 puts "--rerun <n> Set the number of iterations of the benchmark that" 267 puts " contribute to the measured run time. Default is #{$rerun}." 268 puts "--inner <n> Set the number of inner (per-runtime-invocation)" 269 puts " iterations. Default is #{$inner}." 270 puts "--outer <n> Set the number of runtime invocations for each benchmark." 271 puts " Default is #{$outer}." 272 puts "--warmup <n> Set the number of warm-up runs per invocation. Default" 273 puts " is #{$warmup}." 274 puts "--timing-mode Set the way that bencher measures time. Possible values" 275 puts " are 'preciseTime' and 'date'. Default is 'preciseTime'." 276 puts "--force-vm-kind Turn off auto-detection of VM kind, and assume that it is" 277 puts " the one specified. Valid arguments are 'jsc' or" 278 puts " 'DumpRenderTree'." 279 puts "--force-vm-copy Force VM builds to be copied to bencher's working directory." 280 puts " This may reduce pathologies resulting from path names." 281 puts "--dont-copy-vms Don't copy VMs even when doing a remote benchmarking run;" 282 puts " instead assume that they are already there." 283 puts "--v8-only Only run V8." 284 puts "--sunspider-only Only run SunSpider." 285 puts "--kraken-only Only run Kraken." 286 puts "--exclude-v8 Exclude V8 (only run SunSpider and Kraken)." 287 puts "--exclude-sunspider Exclude SunSpider (only run V8 and Kraken)." 288 puts "--exclude-kraken Exclude Kraken (only run SunSpider and V8)." 289 puts "--benchmarks Only run benchmarks matching the given regular expression." 290 puts "--measure-gc Turn off manual calls to gc(), so that GC time is measured." 291 puts " Works best with large values of --inner. You can also say" 292 puts " --measure-gc <conf>, which turns this on for one" 293 puts " configuration only." 294 puts "--verbose or -v Print more stuff." 295 puts "--brief Print only the final result for each VM." 296 puts "--silent Don't print progress. This might slightly reduce some" 297 puts " performance perturbation." 298 puts "--remote <sshhosts> Performance performance measurements remotely, on the given" 299 puts " SSH host(s). Easiest way to use this is to specify the SSH" 300 puts " user@host string. However, you can also supply a comma-" 301 puts " separated list of SSH hosts. Alternatively, you can use this" 302 puts " option multiple times to specify multiple hosts. This" 303 puts " automatically copies the WebKit release builds of the VMs" 304 puts " you specified to all of the hosts." 305 puts "--ssh-options Pass additional options to SSH." 306 puts "--local Also do a local benchmark run even when doing --remote." 307 puts "--prepare-only Only prepare the bencher runscript (a shell script that" 308 puts " invokes the VMs to run benchmarks) but don't run it." 309 puts "--analyze Only read the output of the runscript but don't do anything" 310 puts " else. This requires passing the same arguments to bencher" 311 puts " that you passed when running --prepare-only." 312 puts "--help or -h Display this message." 313 puts 314 puts "Example:" 315 puts "bencher TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc" 316 exit 1 317 end 318 319 def fail(reason) 320 if reason.respond_to? :backtrace 321 puts "FAILED: #{reason}" 322 puts "Stack trace:" 323 puts reason.backtrace.join("\n") 324 else 325 puts "FAILED: #{reason}" 326 end 327 smallUsage 328 end 329 330 def quickFail(r1,r2) 331 $stderr.puts "#{$0}: #{r1}" 332 puts 333 fail(r2) 334 end 335 336 def intArg(argName,arg,min,max) 337 result=arg.to_i 338 unless result.to_s == arg 339 quickFail("Expected an integer value for #{argName}, but got #{arg}.", 340 "Invalid argument for command-line option") 341 end 342 if min and result<min 343 quickFail("Argument for #{argName} cannot be smaller than #{min}.", 344 "Invalid argument for command-line option") 345 end 346 if max and result>max 347 quickFail("Argument for #{argName} cannot be greater than #{max}.", 348 "Invalid argument for command-line option") 349 end 350 result 351 end 352 353 def computeMean(array) 354 sum=0.0 355 array.each { 356 | value | 357 sum += value 358 } 359 sum/array.length 360 end 361 362 def computeGeometricMean(array) 363 mult=1.0 364 array.each { 365 | value | 366 mult*=value 367 } 368 mult**(1.0/array.length) 369 end 370 371 def computeHarmonicMean(array) 372 1.0 / computeMean(array.collect{ | value | 1.0 / value }) 373 end 374 375 def computeStdDev(array) 376 case array.length 377 when 0 378 0.0/0.0 379 when 1 380 0.0 381 else 382 begin 383 mean=computeMean(array) 384 sum=0.0 385 array.each { 386 | value | 387 sum += (value-mean)**2 388 } 389 Math.sqrt(sum/(array.length-1)) 390 rescue 391 0.0/0.0 392 end 393 end 394 end 395 396 class Array 397 def shuffle! 398 size.downto(1) { |n| push delete_at(rand(n)) } 399 self 400 end 401 end 402 403 def inverseBetaRegularized(n) 404 IBR_LOOKUP[n-1] 405 end 406 407 def numToStr(num) 408 "%.4f"%(num.to_f) 409 end 410 411 class NoChange 412 attr_reader :amountFaster 413 414 def initialize(amountFaster) 415 @amountFaster = amountFaster 416 end 417 418 def shortForm 419 " " 420 end 421 422 def longForm 423 " might be #{numToStr(@amountFaster)}x faster" 424 end 425 426 def to_s 427 if @amountFaster < 1.01 428 "" 429 else 430 longForm 431 end 432 end 433 end 434 435 class Faster 436 attr_reader :amountFaster 437 438 def initialize(amountFaster) 439 @amountFaster = amountFaster 440 end 441 442 def shortForm 443 "^" 444 end 445 446 def longForm 447 "^ definitely #{numToStr(@amountFaster)}x faster" 448 end 449 450 def to_s 451 longForm 452 end 453 end 454 455 class Slower 456 attr_reader :amountSlower 457 458 def initialize(amountSlower) 459 @amountSlower = amountSlower 460 end 461 462 def shortForm 463 "!" 464 end 465 466 def longForm 467 "! definitely #{numToStr(@amountSlower)}x slower" 468 end 469 470 def to_s 471 longForm 472 end 473 end 474 475 class MayBeSlower 476 attr_reader :amountSlower 477 478 def initialize(amountSlower) 479 @amountSlower = amountSlower 480 end 481 482 def shortForm 483 "?" 484 end 485 486 def longForm 487 "? might be #{numToStr(@amountSlower)}x slower" 488 end 489 490 def to_s 491 if @amountSlower < 1.01 492 "?" 493 else 494 longForm 495 end 496 end 497 end 498 499 class Stats 500 def initialize 501 @array = [] 502 end 503 504 def add(value) 505 if value.is_a? Stats 506 add(value.array) 507 elsif value.respond_to? :each 508 value.each { 509 | v | 510 add(v) 511 } 512 else 513 @array << value.to_f 514 end 515 end 516 517 def array 518 @array 519 end 520 521 def sum 522 result=0 523 @array.each { 524 | value | 525 result += value 526 } 527 result 528 end 529 530 def min 531 @array.min 532 end 533 534 def max 535 @array.max 536 end 537 538 def size 539 @array.length 540 end 541 542 def mean 543 computeMean(array) 544 end 545 546 def arithmeticMean 547 mean 548 end 549 550 def stdDev 551 computeStdDev(array) 552 end 553 554 def stdErr 555 stdDev/Math.sqrt(size) 556 end 557 558 # Computes a 95% Student's t distribution confidence interval 559 def confInt 560 if size < 2 561 0.0/0.0 562 else 563 raise if size > 1000 564 Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1)) 565 end 566 end 567 568 def lower 569 mean-confInt 570 end 571 572 def upper 573 mean+confInt 574 end 575 576 def geometricMean 577 computeGeometricMean(array) 578 end 579 580 def harmonicMean 581 computeHarmonicMean(array) 582 end 583 584 def compareTo(other) 585 if upper < other.lower 586 Faster.new(other.mean/mean) 587 elsif lower > other.upper 588 Slower.new(mean/other.mean) 589 elsif mean > other.mean 590 MayBeSlower.new(mean/other.mean) 591 else 592 NoChange.new(other.mean/mean) 593 end 594 end 595 596 def to_s 597 "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}" 598 end 599 end 600 601 def doublePuts(out1,out2,msg) 602 out1.puts "#{out2.path}: #{msg}" if $verbosity>=3 603 out2.puts msg 604 end 605 606 class Benchfile < File 607 @@counter = 0 608 609 attr_reader :filename, :basename 610 611 def initialize(name) 612 @basename, @filename = Benchfile.uniqueFilename(name) 613 super(@filename, "w") 614 end 615 616 def self.uniqueFilename(name) 617 if name.is_a? Array 618 basename = name[0] + @@counter.to_s + name[1] 619 else 620 basename = name + @@counter.to_s 621 end 622 filename = BENCH_DATA_PATH + "/" + basename 623 @@counter += 1 624 raise "Benchfile #{filename} already exists" if FileTest.exist?(filename) 625 [basename, filename] 626 end 627 628 def self.create(name) 629 file = Benchfile.new(name) 630 yield file 631 file.close 632 file.basename 633 end 634 end 635 636 $dataFiles={} 637 def ensureFile(key, filename) 638 unless $dataFiles[key] 639 $dataFiles[key] = Benchfile.create(key) { 640 | outp | 641 doublePuts($stderr,outp,IO::read(filename)) 642 } 643 end 644 $dataFiles[key] 645 end 646 647 def emitBenchRunCodeFile(name, plan, benchDataPath, benchPath) 648 case plan.vm.vmType 649 when :jsc 650 Benchfile.create("bencher") { 651 | file | 652 case $timeMode 653 when :preciseTime 654 doublePuts($stderr,file,"function __bencher_curTimeMS() {") 655 doublePuts($stderr,file," return preciseTime()*1000") 656 doublePuts($stderr,file,"}") 657 when :date 658 doublePuts($stderr,file,"function __bencher_curTimeMS() {") 659 doublePuts($stderr,file," return Date.now()") 660 doublePuts($stderr,file,"}") 661 else 662 raise 663 end 664 665 if benchDataPath 666 doublePuts($stderr,file,"load(#{benchDataPath.inspect});") 667 doublePuts($stderr,file,"gc();") 668 doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {") 669 doublePuts($stderr,file," before = __bencher_curTimeMS();") 670 $rerun.times { 671 doublePuts($stderr,file," load(#{benchPath.inspect});") 672 } 673 doublePuts($stderr,file," after = __bencher_curTimeMS();") 674 doublePuts($stderr,file," if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(after-before));"); 675 doublePuts($stderr,file," gc();") unless plan.vm.shouldMeasureGC 676 doublePuts($stderr,file,"}") 677 else 678 doublePuts($stderr,file,"function __bencher_run(__bencher_what) {") 679 doublePuts($stderr,file," var __bencher_before = __bencher_curTimeMS();") 680 $rerun.times { 681 doublePuts($stderr,file," run(__bencher_what);") 682 } 683 doublePuts($stderr,file," var __bencher_after = __bencher_curTimeMS();") 684 doublePuts($stderr,file," return __bencher_after - __bencher_before;") 685 doublePuts($stderr,file,"}") 686 $warmup.times { 687 doublePuts($stderr,file,"__bencher_run(#{benchPath.inspect})") 688 doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC 689 } 690 $inner.times { 691 | innerIndex | 692 doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchPath.inspect}));") 693 doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC 694 } 695 end 696 } 697 when :dumpRenderTree 698 mainCode = Benchfile.create("bencher") { 699 | file | 700 doublePuts($stderr,file,"__bencher_count = 0;") 701 doublePuts($stderr,file,"function __bencher_doNext(result) {") 702 doublePuts($stderr,file," if (__bencher_count >= #{$warmup})") 703 doublePuts($stderr,file," debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);") 704 doublePuts($stderr,file," __bencher_count++;") 705 doublePuts($stderr,file," if (__bencher_count < #{$inner+$warmup})") 706 doublePuts($stderr,file," __bencher_runImpl(__bencher_doNext);") 707 doublePuts($stderr,file," else") 708 doublePuts($stderr,file," quit();") 709 doublePuts($stderr,file,"}") 710 doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);") 711 } 712 713 cssCode = Benchfile.create("bencher-css") { 714 | file | 715 doublePuts($stderr,file,".pass {\n font-weight: bold;\n color: green;\n}\n.fail {\n font-weight: bold;\n color: red;\n}\n\#console {\n white-space: pre-wrap;\n font-family: monospace;\n}") 716 } 717 718 preCode = Benchfile.create("bencher-pre") { 719 | file | 720 doublePuts($stderr,file,"if (window.testRunner) {") 721 doublePuts($stderr,file," if (window.enablePixelTesting) {") 722 doublePuts($stderr,file," testRunner.dumpAsTextWithPixelResults();") 723 doublePuts($stderr,file," } else {") 724 doublePuts($stderr,file," testRunner.dumpAsText();") 725 doublePuts($stderr,file," }") 726 doublePuts($stderr,file,"}") 727 doublePuts($stderr,file,"") 728 doublePuts($stderr,file,"function debug(msg)") 729 doublePuts($stderr,file,"{") 730 doublePuts($stderr,file," var span = document.createElement(\"span\");") 731 doublePuts($stderr,file," document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace") 732 doublePuts($stderr,file," span.innerHTML = msg + '<br />';") 733 doublePuts($stderr,file,"}") 734 doublePuts($stderr,file,"") 735 doublePuts($stderr,file,"function quit() {") 736 doublePuts($stderr,file," testRunner.notifyDone();") 737 doublePuts($stderr,file,"}") 738 doublePuts($stderr,file,"") 739 doublePuts($stderr,file,"__bencher_continuation=null;") 740 doublePuts($stderr,file,"") 741 doublePuts($stderr,file,"function reportResult(result) {") 742 doublePuts($stderr,file," __bencher_continuation(result);") 743 doublePuts($stderr,file,"}") 744 doublePuts($stderr,file,"") 745 doublePuts($stderr,file,"function __bencher_runImpl(continuation) {") 746 doublePuts($stderr,file," function doit() {") 747 doublePuts($stderr,file," document.getElementById(\"frameparent\").innerHTML = \"\";") 748 doublePuts($stderr,file," document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";") 749 doublePuts($stderr,file," var testFrame = document.getElementById(\"testframe\");") 750 doublePuts($stderr,file," testFrame.contentDocument.open();") 751 doublePuts($stderr,file," testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");") 752 if benchDataPath 753 doublePuts($stderr,file," testFrame.contentDocument.write(\"<script src=\\\"#{benchDataPath}\\\"></script>\");") 754 end 755 doublePuts($stderr,file," testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">__bencher_before = Date.now();</script><script src=\\\"#{benchPath}\\\"></script><script type=\\\"text/javascript\\\">window.parent.reportResult(Date.now() - __bencher_before);</script></body></html>\");") 756 doublePuts($stderr,file," testFrame.contentDocument.close();") 757 doublePuts($stderr,file," }") 758 doublePuts($stderr,file," __bencher_continuation = continuation;") 759 doublePuts($stderr,file," window.setTimeout(doit, 10);") 760 doublePuts($stderr,file,"}") 761 } 762 763 Benchfile.create(["bencher-htmldoc",".html"]) { 764 | file | 765 doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>") 766 } 767 else 768 raise 769 end 770 end 771 772 def emitBenchRunCode(name, plan, benchDataPath, benchPath) 773 plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchDataPath, benchPath)) 774 end 775 776 def planForDescription(plans, benchFullname, vmName, iteration) 777 raise unless benchFullname =~ /\// 778 suiteName = $~.pre_match 779 benchName = $~.post_match 780 result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration} 781 raise unless result.size == 1 782 result[0] 783 end 784 785 class ParsedResult 786 attr_reader :plan, :innerIndex, :time 787 788 def initialize(plan, innerIndex, time) 789 @plan = plan 790 @innerIndex = innerIndex 791 @time = time 792 793 raise unless @plan.is_a? BenchPlan 794 raise unless @innerIndex.is_a? Integer 795 raise unless @time.is_a? Numeric 796 end 797 798 def benchmark 799 plan.benchmark 800 end 801 802 def suite 803 plan.suite 804 end 805 806 def vm 807 plan.vm 808 end 809 810 def outerIndex 811 plan.iteration 812 end 813 814 def self.parse(plans, string) 815 if string =~ /([a-zA-Z0-9\/-]+): ([a-zA-Z0-9_# ]+): ([0-9]+): ([0-9]+): Time: / 816 benchFullname = $1 817 vmName = $2 818 outerIndex = $3.to_i 819 innerIndex = $4.to_i 820 time = $~.post_match.to_f 821 ParsedResult.new(planForDescription(plans, benchFullname, vmName, outerIndex), innerIndex, time) 822 else 823 nil 824 end 825 end 826 end 827 828 class VM 829 def initialize(origPath, name, nameKind, svnRevision) 830 @origPath = origPath.to_s 831 @path = origPath.to_s 832 @name = name 833 @nameKind = nameKind 834 835 if $forceVMKind 836 @vmType = $forceVMKind 837 else 838 if @origPath =~ /DumpRenderTree$/ 839 @vmType = :dumpRenderTree 840 else 841 @vmType = :jsc 842 end 843 end 844 845 @svnRevision = svnRevision 846 847 # Try to detect information about the VM. 848 if path =~ /\/WebKitBuild\/Release\/([a-zA-Z]+)$/ 849 @checkoutPath = $~.pre_match 850 # FIXME: Use some variant of this: 851 # <bdash> def retrieve_revision 852 # <bdash> `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i 853 # <bdash> end 854 unless @svnRevision 855 begin 856 Dir.chdir(@checkoutPath) { 857 $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2 858 IO.popen("svn info", "r") { 859 | inp | 860 inp.each_line { 861 | line | 862 if line =~ /Revision: ([0-9]+)/ 863 @svnRevision = $1 864 end 865 } 866 } 867 } 868 unless @svnRevision 869 $stderr.puts "Warning: running svn info for #{name} silently failed." 870 end 871 rescue => e 872 # Failed to detect svn revision. 873 $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}" 874 end 875 end 876 else 877 $stderr.puts "Warning: could not identify checkout location for #{name}" 878 end 879 880 if @path =~ /\/Release\/([a-zA-Z]+)$/ 881 @libPath, @relativeBinPath = $~.pre_match+"/Release", "./#{$1}" 882 elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/ 883 @libPath = $~.pre_match 884 elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/ 885 @libPath, @relativeBinPath = $~.pre_match, $&[1..-1] 886 end 887 end 888 889 def canCopyIntoBenchPath 890 if @libPath and @relativeBinPath 891 true 892 else 893 false 894 end 895 end 896 897 def copyIntoBenchPath 898 raise unless canCopyIntoBenchPath 899 basename, filename = Benchfile.uniqueFilename("vm") 900 raise unless Dir.mkdir(filename) 901 cmd = "cp -a #{@libPath.inspect}/* #{filename.inspect}" 902 $stderr.puts ">> #{cmd}" if $verbosity>=2 903 raise unless system(cmd) 904 @path = "#{basename}/#{@relativeBinPath}" 905 @libPath = basename 906 end 907 908 def to_s 909 @name 910 end 911 912 def name 913 @name 914 end 915 916 def shouldMeasureGC 917 $measureGC == true or ($measureGC == name) 918 end 919 920 def origPath 921 @origPath 922 end 923 924 def path 925 @path 926 end 927 928 def nameKind 929 @nameKind 930 end 931 932 def vmType 933 @vmType 934 end 935 936 def checkoutPath 937 @checkoutPath 938 end 939 940 def svnRevision 941 @svnRevision 942 end 943 944 def printFunction 945 case @vmType 946 when :jsc 947 "print" 948 when :dumpRenderTree 949 "debug" 950 else 951 raise @vmType 952 end 953 end 954 955 def emitRunCode(fileToRun) 956 myLibPath = @libPath 957 myLibPath = "" unless myLibPath 958 $script.puts "export DYLD_LIBRARY_PATH=#{myLibPath.to_s.inspect}" 959 $script.puts "export DYLD_FRAMEWORK_PATH=#{myLibPath.to_s.inspect}" 960 $script.puts "#{path} #{fileToRun}" 961 end 962 end 963 964 class StatsAccumulator 965 def initialize 966 @stats = [] 967 ($outer*$inner).times { 968 @stats << Stats.new 969 } 970 end 971 972 def statsForIteration(outerIteration, innerIteration) 973 @stats[outerIteration*$inner + innerIteration] 974 end 975 976 def stats 977 result = Stats.new 978 @stats.each { 979 | stat | 980 result.add(yield stat) 981 } 982 result 983 end 984 985 def geometricMeanStats 986 stats { 987 | stat | 988 stat.geometricMean 989 } 990 end 991 992 def arithmeticMeanStats 993 stats { 994 | stat | 995 stat.arithmeticMean 996 } 997 end 998 end 999 1000 module Benchmark 1001 attr_accessor :benchmarkSuite 1002 attr_reader :name 1003 1004 def fullname 1005 benchmarkSuite.name + "/" + name 1006 end 1007 1008 def to_s 1009 fullname 1010 end 1011 end 1012 1013 class SunSpiderBenchmark 1014 include Benchmark 1015 1016 def initialize(name) 1017 @name = name 1018 end 1019 1020 def emitRunCode(plan) 1021 emitBenchRunCode(fullname, plan, nil, ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")) 1022 end 1023 end 1024 1025 class V8Benchmark 1026 include Benchmark 1027 1028 def initialize(name) 1029 @name = name 1030 end 1031 1032 def emitRunCode(plan) 1033 emitBenchRunCode(fullname, plan, nil, ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")) 1034 end 1035 end 1036 1037 class KrakenBenchmark 1038 include Benchmark 1039 1040 def initialize(name) 1041 @name = name 1042 end 1043 1044 def emitRunCode(plan) 1045 emitBenchRunCode(fullname, plan, ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js"), ensureFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")) 1046 end 1047 end 1048 1049 class BenchmarkSuite 1050 def initialize(name, path, preferredMean) 1051 @name = name 1052 @path = path 1053 @preferredMean = preferredMean 1054 @benchmarks = [] 1055 end 1056 1057 def name 1058 @name 1059 end 1060 1061 def to_s 1062 @name 1063 end 1064 1065 def path 1066 @path 1067 end 1068 1069 def add(benchmark) 1070 if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern 1071 benchmark.benchmarkSuite = self 1072 @benchmarks << benchmark 1073 end 1074 end 1075 1076 def benchmarks 1077 @benchmarks 1078 end 1079 1080 def benchmarkForName(name) 1081 result = @benchmarks.select{|v| v.name == name} 1082 raise unless result.length == 1 1083 result[0] 1084 end 1085 1086 def empty? 1087 @benchmarks.empty? 1088 end 1089 1090 def retain_if 1091 @benchmarks.delete_if { 1092 | benchmark | 1093 not yield benchmark 1094 } 1095 end 1096 1097 def preferredMean 1098 @preferredMean 1099 end 1100 1101 def computeMean(stat) 1102 stat.send @preferredMean 1103 end 1104 end 1105 1106 class BenchRunPlan 1107 def initialize(benchmark, vm, iteration) 1108 @benchmark = benchmark 1109 @vm = vm 1110 @iteration = iteration 1111 end 1112 1113 def benchmark 1114 @benchmark 1115 end 1116 1117 def suite 1118 @benchmark.benchmarkSuite 1119 end 1120 1121 def vm 1122 @vm 1123 end 1124 1125 def iteration 1126 @iteration 1127 end 1128 1129 def emitRunCode 1130 @benchmark.emitRunCode(self) 1131 end 1132 end 1133 1134 class BenchmarkOnVM 1135 def initialize(benchmark, suiteOnVM) 1136 @benchmark = benchmark 1137 @suiteOnVM = suiteOnVM 1138 @stats = Stats.new 1139 end 1140 1141 def to_s 1142 "#{@benchmark} on #{@suiteOnVM.vm}" 1143 end 1144 1145 def benchmark 1146 @benchmark 1147 end 1148 1149 def vm 1150 @suiteOnVM.vm 1151 end 1152 1153 def vmStats 1154 @suiteOnVM.vmStats 1155 end 1156 1157 def suite 1158 @benchmark.benchmarkSuite 1159 end 1160 1161 def suiteOnVM 1162 @suiteOnVM 1163 end 1164 1165 def stats 1166 @stats 1167 end 1168 1169 def parseResult(result) 1170 raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm 1171 raise unless result.benchmark == @benchmark 1172 @stats.add(result.time) 1173 end 1174 end 1175 1176 class SuiteOnVM < StatsAccumulator 1177 def initialize(vm, vmStats, suite) 1178 super() 1179 @vm = vm 1180 @vmStats = vmStats 1181 @suite = suite 1182 1183 raise unless @vm.is_a? VM 1184 raise unless @vmStats.is_a? StatsAccumulator 1185 raise unless @suite.is_a? BenchmarkSuite 1186 end 1187 1188 def to_s 1189 "#{@suite} on #{@vm}" 1190 end 1191 1192 def suite 1193 @suite 1194 end 1195 1196 def vm 1197 @vm 1198 end 1199 1200 def vmStats 1201 raise unless @vmStats 1202 @vmStats 1203 end 1204 end 1205 1206 class BenchPlan 1207 def initialize(benchmarkOnVM, iteration) 1208 @benchmarkOnVM = benchmarkOnVM 1209 @iteration = iteration 1210 end 1211 1212 def to_s 1213 "#{@benchmarkOnVM} \##{@iteration+1}" 1214 end 1215 1216 def benchmarkOnVM 1217 @benchmarkOnVM 1218 end 1219 1220 def benchmark 1221 @benchmarkOnVM.benchmark 1222 end 1223 1224 def suite 1225 @benchmarkOnVM.suite 1226 end 1227 1228 def vm 1229 @benchmarkOnVM.vm 1230 end 1231 1232 def iteration 1233 @iteration 1234 end 1235 1236 def parseResult(result) 1237 raise unless result.plan == self 1238 @benchmarkOnVM.parseResult(result) 1239 @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time) 1240 @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time) 1241 end 1242 end 1243 1244 def lpad(str,chars) 1245 if str.length>chars 1246 str 1247 else 1248 "%#{chars}s"%(str) 1249 end 1250 end 1251 1252 def rpad(str,chars) 1253 while str.length<chars 1254 str+=" " 1255 end 1256 str 1257 end 1258 1259 def center(str,chars) 1260 while str.length<chars 1261 str+=" " 1262 if str.length<chars 1263 str=" "+str 1264 end 1265 end 1266 str 1267 end 1268 1269 def statsToStr(stats) 1270 if $inner*$outer == 1 1271 string = numToStr(stats.mean) 1272 raise unless string =~ /\./ 1273 left = $~.pre_match 1274 right = $~.post_match 1275 lpad(left,12)+"."+rpad(right,9) 1276 else 1277 lpad(numToStr(stats.mean),11)+"+-"+rpad(numToStr(stats.confInt),9) 1278 end 1279 end 1280 1281 def plural(num) 1282 if num == 1 1283 "" 1284 else 1285 "s" 1286 end 1287 end 1288 1289 def wrap(str, columns) 1290 array = str.split 1291 result = "" 1292 curLine = array.shift 1293 array.each { 1294 | curStr | 1295 if (curLine + " " + curStr).size > columns 1296 result += curLine + "\n" 1297 curLine = curStr 1298 else 1299 curLine += " " + curStr 1300 end 1301 } 1302 result + curLine + "\n" 1303 end 1304 1305 def runAndGetResults 1306 results = nil 1307 Dir.chdir(BENCH_DATA_PATH) { 1308 IO.popen("sh ./runscript", "r") { 1309 | inp | 1310 results = inp.read 1311 } 1312 raise "Script did not complete correctly: #{$?}" unless $?.success? 1313 } 1314 raise unless results 1315 results 1316 end 1317 1318 def parseAndDisplayResults(results) 1319 vmStatses = [] 1320 $vms.each { 1321 vmStatses << StatsAccumulator.new 1322 } 1323 1324 suitesOnVMs = [] 1325 suitesOnVMsForSuite = {} 1326 $suites.each { 1327 | suite | 1328 suitesOnVMsForSuite[suite] = [] 1329 } 1330 suitesOnVMsForVM = {} 1331 $vms.each { 1332 | vm | 1333 suitesOnVMsForVM[vm] = [] 1334 } 1335 1336 benchmarksOnVMs = [] 1337 benchmarksOnVMsForBenchmark = {} 1338 $benchmarks.each { 1339 | benchmark | 1340 benchmarksOnVMsForBenchmark[benchmark] = [] 1341 } 1342 1343 $vms.each_with_index { 1344 | vm, vmIndex | 1345 vmStats = vmStatses[vmIndex] 1346 $suites.each { 1347 | suite | 1348 suiteOnVM = SuiteOnVM.new(vm, vmStats, suite) 1349 suitesOnVMs << suiteOnVM 1350 suitesOnVMsForSuite[suite] << suiteOnVM 1351 suitesOnVMsForVM[vm] << suiteOnVM 1352 suite.benchmarks.each { 1353 | benchmark | 1354 benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM) 1355 benchmarksOnVMs << benchmarkOnVM 1356 benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM 1357 } 1358 } 1359 } 1360 1361 plans = [] 1362 benchmarksOnVMs.each { 1363 | benchmarkOnVM | 1364 $outer.times { 1365 | iteration | 1366 plans << BenchPlan.new(benchmarkOnVM, iteration) 1367 } 1368 } 1369 1370 hostname = nil 1371 hwmodel = nil 1372 results.each_line { 1373 | line | 1374 line.chomp! 1375 if line =~ /HOSTNAME:([^.]+)/ 1376 hostname = $1 1377 elsif line =~ /HARDWARE:hw\.model: / 1378 hwmodel = $~.post_match.chomp 1379 else 1380 result = ParsedResult.parse(plans, line.chomp) 1381 if result 1382 result.plan.parseResult(result) 1383 end 1384 end 1385 } 1386 1387 # Compute the geomean of the preferred means of results on a SuiteOnVM 1388 overallResults = [] 1389 $vms.each { 1390 | vm | 1391 result = Stats.new 1392 $outer.times { 1393 | outerIndex | 1394 $inner.times { 1395 | innerIndex | 1396 curResult = Stats.new 1397 suitesOnVMsForVM[vm].each { 1398 | suiteOnVM | 1399 # For a given iteration, suite, and VM, compute the suite's preferred mean 1400 # over the data collected for all benchmarks in that suite. We'll have one 1401 # sample per benchmark. For example on V8 this will be the geomean of 1 1402 # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for 1403 # splay. 1404 curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex))) 1405 } 1406 1407 # curResult now holds 1 sample for each of the means computed in the above 1408 # loop. Compute the geomean over this, and store it. 1409 result.add(curResult.geometricMean) 1410 } 1411 } 1412 1413 # $overallResults will have a Stats for each VM. That Stats object will hold 1414 # $inner*$outer geomeans, allowing us to compute the arithmetic mean and 1415 # confidence interval of the geomeans of preferred means. Convoluted, but 1416 # useful and probably sound. 1417 overallResults << result 1418 } 1419 1420 if $verbosity >= 2 1421 benchmarksOnVMs.each { 1422 | benchmarkOnVM | 1423 $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}" 1424 } 1425 1426 $vms.each_with_index { 1427 | vm, vmIndex | 1428 vmStats = vmStatses[vmIndex] 1429 $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}" 1430 $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}" 1431 } 1432 end 1433 1434 reportName = 1435 (if ($vms.collect { 1436 | vm | 1437 vm.nameKind 1438 }.index :auto) 1439 "" 1440 else 1441 $vms.collect { 1442 | vm | 1443 vm.to_s 1444 }.join("_") + "_" 1445 end) + 1446 ($suites.collect { 1447 | suite | 1448 suite.to_s 1449 }.join("")) + "_" + 1450 (if hostname 1451 hostname + "_" 1452 else 1453 "" 1454 end)+ 1455 (begin 1456 time = Time.now 1457 "%04d%02d%02d_%02d%02d" % 1458 [ time.year, time.month, time.day, 1459 time.hour, time.min ] 1460 end) + 1461 "_benchReport.txt" 1462 1463 unless $brief 1464 puts "Generating benchmark report at #{reportName}" 1465 end 1466 1467 outp = $stdout 1468 begin 1469 outp = File.open(reportName,"w") 1470 rescue => e 1471 $stderr.puts "Error: could not save report to #{reportName}: #{e}" 1472 $stderr.puts 1473 end 1474 1475 def createVMsString 1476 result = "" 1477 result += " " if $suites.size > 1 1478 result += rpad("", $benchpad) 1479 result += " " 1480 $vms.size.times { 1481 | index | 1482 if index != 0 1483 result += " "+NoChange.new(0).shortForm 1484 end 1485 result += lpad(center($vms[index].name, 9+9+2), 11+9+2) 1486 } 1487 result += " " 1488 if $vms.size >= 3 1489 result += center("#{$vms[-1].name} v. #{$vms[0].name}",26) 1490 elsif $vms.size >= 2 1491 result += " "*26 1492 end 1493 result 1494 end 1495 1496 columns = [createVMsString.size, 78].max 1497 1498 outp.print "Benchmark report for " 1499 if $suites.size == 1 1500 outp.print $suites[0].to_s 1501 elsif $suites.size == 2 1502 outp.print "#{$suites[0]} and #{$suites[1]}" 1503 else 1504 outp.print "#{$suites[0..-2].join(', ')}, and #{$suites[-1]}" 1505 end 1506 if hostname 1507 outp.print " on #{hostname}" 1508 end 1509 if hwmodel 1510 outp.print " (#{hwmodel})" 1511 end 1512 outp.puts "." 1513 outp.puts 1514 1515 # This looks stupid; revisit later. 1516 if false 1517 $suites.each { 1518 | suite | 1519 outp.puts "#{suite} at #{suite.path}" 1520 } 1521 1522 outp.puts 1523 end 1524 1525 outp.puts "VMs tested:" 1526 $vms.each { 1527 | vm | 1528 outp.print "\"#{vm.name}\" at #{vm.origPath}" 1529 if vm.svnRevision 1530 outp.print " (r#{vm.svnRevision})" 1531 end 1532 outp.puts 1533 } 1534 1535 outp.puts 1536 1537 outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+ 1538 "with #{$outer} VM invocation#{plural($outer)} per benchmark."+ 1539 (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+ 1540 "total time of those iterations, for each sample.") 1541 else "" end)+ 1542 (if $measureGC == true then (" No manual garbage collection invocations were "+ 1543 "emitted.") 1544 elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+ 1545 "all VMs except #{$measureGC}.") 1546 else (" Emitted a call to gc() between sample measurements.") end)+ 1547 (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+ 1548 "began with the very first iteration.") 1549 else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+ 1550 "invocation for warm-up.") end)+ 1551 (case $timeMode 1552 when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+ 1553 "microsecond-level timing.") 1554 when :date then (" Used the portable Date.now() method to get millisecond-"+ 1555 "level timing.") 1556 else raise end)+ 1557 " Reporting benchmark execution times with 95% confidence "+ 1558 "intervals in milliseconds.", 1559 columns) 1560 1561 outp.puts 1562 1563 def printVMs(outp) 1564 outp.puts createVMsString 1565 end 1566 1567 def summaryStats(outp, accumulators, name, &proc) 1568 outp.print " " if $suites.size > 1 1569 outp.print rpad(name, $benchpad) 1570 outp.print " " 1571 accumulators.size.times { 1572 | index | 1573 if index != 0 1574 outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm 1575 end 1576 outp.print statsToStr(accumulators[index].stats(&proc)) 1577 } 1578 if accumulators.size>=2 1579 outp.print(" "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm) 1580 end 1581 outp.puts 1582 end 1583 1584 def meanName(currentMean, preferredMean) 1585 result = "<#{currentMean}>" 1586 if "#{currentMean}Mean" == preferredMean.to_s 1587 result += " *" 1588 end 1589 result 1590 end 1591 1592 def allSummaryStats(outp, accumulators, preferredMean) 1593 summaryStats(outp, accumulators, meanName("arithmetic", preferredMean)) { 1594 | stat | 1595 stat.arithmeticMean 1596 } 1597 1598 summaryStats(outp, accumulators, meanName("geometric", preferredMean)) { 1599 | stat | 1600 stat.geometricMean 1601 } 1602 1603 summaryStats(outp, accumulators, meanName("harmonic", preferredMean)) { 1604 | stat | 1605 stat.harmonicMean 1606 } 1607 end 1608 1609 $suites.each { 1610 | suite | 1611 printVMs(outp) 1612 if $suites.size > 1 1613 outp.puts "#{suite.name}:" 1614 else 1615 outp.puts 1616 end 1617 suite.benchmarks.each { 1618 | benchmark | 1619 outp.print " " if $suites.size > 1 1620 outp.print rpad(benchmark.name, $benchpad) 1621 outp.print " " 1622 myConfigs = benchmarksOnVMsForBenchmark[benchmark] 1623 myConfigs.size.times { 1624 | index | 1625 if index != 0 1626 outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm 1627 end 1628 outp.print statsToStr(myConfigs[index].stats) 1629 } 1630 if $vms.size>=2 1631 outp.print(" "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s) 1632 end 1633 outp.puts 1634 } 1635 outp.puts 1636 allSummaryStats(outp, suitesOnVMsForSuite[suite], suite.preferredMean) 1637 outp.puts if $suites.size > 1 1638 } 1639 1640 if $suites.size > 1 1641 printVMs(outp) 1642 outp.puts "All benchmarks:" 1643 allSummaryStats(outp, vmStatses, nil) 1644 1645 outp.puts 1646 printVMs(outp) 1647 outp.puts "Geomean of preferred means:" 1648 outp.print " " 1649 outp.print rpad("<scaled-result>", $benchpad) 1650 outp.print " " 1651 $vms.size.times { 1652 | index | 1653 if index != 0 1654 outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm 1655 end 1656 outp.print statsToStr(overallResults[index]) 1657 } 1658 if overallResults.size>=2 1659 outp.print(" "+overallResults[-1].compareTo(overallResults[0]).longForm) 1660 end 1661 outp.puts 1662 end 1663 outp.puts 1664 1665 if outp != $stdout 1666 outp.close 1667 end 1668 1669 if outp != $stdout and not $brief 1670 puts 1671 File.open(reportName) { 1672 | inp | 1673 puts inp.read 1674 } 1675 end 1676 1677 if $brief 1678 puts(overallResults.collect{|stats| stats.mean}.join("\t")) 1679 puts(overallResults.collect{|stats| stats.confInt}.join("\t")) 1680 end 1681 1682 1683 end 1684 1685 begin 1686 $sawBenchOptions = false 1687 1688 def resetBenchOptionsIfNecessary 1689 unless $sawBenchOptions 1690 $includeSunSpider = false 1691 $includeV8 = false 1692 $includeKraken = false 1693 $sawBenchOptions = true 1694 end 1695 end 1696 1697 GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT], 1698 ['--inner', GetoptLong::REQUIRED_ARGUMENT], 1699 ['--outer', GetoptLong::REQUIRED_ARGUMENT], 1700 ['--warmup', GetoptLong::REQUIRED_ARGUMENT], 1701 ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT], 1702 ['--sunspider-only', GetoptLong::NO_ARGUMENT], 1703 ['--v8-only', GetoptLong::NO_ARGUMENT], 1704 ['--kraken-only', GetoptLong::NO_ARGUMENT], 1705 ['--exclude-sunspider', GetoptLong::NO_ARGUMENT], 1706 ['--exclude-v8', GetoptLong::NO_ARGUMENT], 1707 ['--exclude-kraken', GetoptLong::NO_ARGUMENT], 1708 ['--sunspider', GetoptLong::NO_ARGUMENT], 1709 ['--v8', GetoptLong::NO_ARGUMENT], 1710 ['--kraken', GetoptLong::NO_ARGUMENT], 1711 ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT], 1712 ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT], 1713 ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT], 1714 ['--force-vm-copy', GetoptLong::NO_ARGUMENT], 1715 ['--dont-copy-vms', GetoptLong::NO_ARGUMENT], 1716 ['--verbose', '-v', GetoptLong::NO_ARGUMENT], 1717 ['--brief', GetoptLong::NO_ARGUMENT], 1718 ['--silent', GetoptLong::NO_ARGUMENT], 1719 ['--remote', GetoptLong::REQUIRED_ARGUMENT], 1720 ['--local', GetoptLong::NO_ARGUMENT], 1721 ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT], 1722 ['--slave', GetoptLong::NO_ARGUMENT], 1723 ['--prepare-only', GetoptLong::NO_ARGUMENT], 1724 ['--analyze', GetoptLong::REQUIRED_ARGUMENT], 1725 ['--vms', GetoptLong::REQUIRED_ARGUMENT], 1726 ['--help', '-h', GetoptLong::NO_ARGUMENT]).each { 1727 | opt, arg | 1728 case opt 1729 when '--rerun' 1730 $rerun = intArg(opt,arg,1,nil) 1731 when '--inner' 1732 $inner = intArg(opt,arg,1,nil) 1733 when '--outer' 1734 $outer = intArg(opt,arg,1,nil) 1735 when '--warmup' 1736 $warmup = intArg(opt,arg,0,nil) 1737 when '--timing-mode' 1738 if arg.upcase == "PRECISETIME" 1739 $timeMode = :preciseTime 1740 elsif arg.upcase == "DATE" 1741 $timeMode = :date 1742 elsif arg.upcase == "AUTO" 1743 $timeMode = :auto 1744 else 1745 quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.", 1746 "Invalid argument for command-line option") 1747 end 1748 when '--force-vm-kind' 1749 if arg.upcase == "JSC" 1750 $forceVMKind = :jsc 1751 elsif arg.upcase == "DUMPRENDERTREE" 1752 $forceVMKind = :dumpRenderTree 1753 elsif arg.upcase == "AUTO" 1754 $forceVMKind = nil 1755 else 1756 quickFail("Expected either 'jsc' or 'DumpRenderTree' for --force-vm-kind, but got '#{arg}'.", 1757 "Invalid argument for command-line option") 1758 end 1759 when '--force-vm-copy' 1760 $needToCopyVMs = true 1761 when '--dont-copy-vms' 1762 $dontCopyVMs = true 1763 when '--sunspider-only' 1764 $includeV8 = false 1765 $includeKraken = false 1766 when '--v8-only' 1767 $includeSunSpider = false 1768 $includeKraken = false 1769 when '--kraken-only' 1770 $includeSunSpider = false 1771 $includeV8 = false 1772 when '--exclude-sunspider' 1773 $includeSunSpider = false 1774 when '--exclude-v8' 1775 $includeV8 = false 1776 when '--exclude-kraken' 1777 $includeKraken = false 1778 when '--sunspider' 1779 resetBenchOptionsIfNecessary 1780 $includeSunSpider = true 1781 when '--v8' 1782 resetBenchOptionsIfNecessary 1783 $includeV8 = true 1784 when '--kraken' 1785 resetBenchOptionsIfNecessary 1786 $includeKraken = true 1787 when '--benchmarks' 1788 $benchmarkPattern = Regexp.new(arg) 1789 when '--measure-gc' 1790 if arg == '' 1791 $measureGC = true 1792 else 1793 $measureGC = arg 1794 end 1795 when '--verbose' 1796 $verbosity += 1 1797 when '--brief' 1798 $brief = true 1799 when '--silent' 1800 $silent = true 1801 when '--remote' 1802 $remoteHosts += arg.split(',') 1803 $needToCopyVMs = true 1804 when '--ssh-options' 1805 $sshOptions << arg 1806 when '--local' 1807 $alsoLocal = true 1808 when '--prepare-only' 1809 $run = false 1810 when '--analyze' 1811 $prepare = false 1812 $run = false 1813 $analyze << arg 1814 when '--help' 1815 usage 1816 else 1817 raise "bad option: #{opt}" 1818 end 1819 } 1820 1821 # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option. 1822 if $dontCopyVMs 1823 $needToCopyVMs = false 1824 end 1825 1826 SUNSPIDER = BenchmarkSuite.new("SunSpider", SUNSPIDER_PATH, :arithmeticMean) 1827 ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees", 1828 "access-fannkuch", "access-nbody", "access-nsieve", 1829 "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and", 1830 "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes", 1831 "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb", 1832 "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna", 1833 "string-base64", "string-fasta", "string-tagcloud", 1834 "string-unpack-code", "string-validate-input"].each { 1835 | name | 1836 SUNSPIDER.add SunSpiderBenchmark.new(name) 1837 } 1838 1839 V8 = BenchmarkSuite.new("V8", V8_PATH, :geometricMean) 1840 ["crypto", "deltablue", "earley-boyer", "raytrace", 1841 "regexp", "richards", "splay"].each { 1842 | name | 1843 V8.add V8Benchmark.new(name) 1844 } 1845 1846 KRAKEN = BenchmarkSuite.new("Kraken", KRAKEN_PATH, :arithmeticMean) 1847 ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft", 1848 "audio-oscillator", "imaging-darkroom", "imaging-desaturate", 1849 "imaging-gaussian-blur", "json-parse-financial", 1850 "json-stringify-tinderbox", "stanford-crypto-aes", 1851 "stanford-crypto-ccm", "stanford-crypto-pbkdf2", 1852 "stanford-crypto-sha256-iterative"].each { 1853 | name | 1854 KRAKEN.add KrakenBenchmark.new(name) 1855 } 1856 1857 ARGV.each { 1858 | vm | 1859 if vm =~ /([a-zA-Z0-9_ ]+):/ 1860 name = $1 1861 nameKind = :given 1862 vm = $~.post_match 1863 else 1864 name = "Conf\##{$vms.length+1}" 1865 nameKind = :auto 1866 end 1867 $stderr.puts "#{name}: #{vm}" if $verbosity >= 1 1868 $vms << VM.new(Pathname.new(vm).realpath, name, nameKind, nil) 1869 } 1870 1871 if $vms.empty? 1872 quickFail("Please specify at least on configuraiton on the command line.", 1873 "Insufficient arguments") 1874 end 1875 1876 $vms.each { 1877 | vm | 1878 if vm.vmType != :jsc and $timeMode != :date 1879 $timeMode = :date 1880 $stderr.puts "Warning: using Date.now() instead of preciseTime() because #{vm} doesn't support the latter." 1881 end 1882 } 1883 1884 if FileTest.exist? BENCH_DATA_PATH 1885 cmd = "rm -rf #{BENCH_DATA_PATH}" 1886 $stderr.puts ">> #{cmd}" if $verbosity >= 2 1887 raise unless system cmd 1888 end 1889 1890 Dir.mkdir BENCH_DATA_PATH 1891 1892 if $needToCopyVMs 1893 canCopyIntoBenchPath = true 1894 $vms.each { 1895 | vm | 1896 canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath 1897 } 1898 1899 if canCopyIntoBenchPath 1900 $vms.each { 1901 | vm | 1902 $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..." 1903 vm.copyIntoBenchPath 1904 } 1905 $stderr.puts "All VMs are in place." 1906 else 1907 $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it." 1908 end 1909 end 1910 1911 if $measureGC and $measureGC != true 1912 found = false 1913 $vms.each { 1914 | vm | 1915 if vm.name == $measureGC 1916 found = true 1917 end 1918 } 1919 unless found 1920 $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}" 1921 end 1922 end 1923 1924 if $outer*$inner == 1 1925 $stderr.puts "Warning: will only collect one sample per benchmark/VM. Confidence interval calculation will fail." 1926 end 1927 1928 $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1 1929 1930 $suites = [] 1931 1932 if $includeSunSpider and not SUNSPIDER.empty? 1933 $suites << SUNSPIDER 1934 end 1935 1936 if $includeV8 and not V8.empty? 1937 $suites << V8 1938 end 1939 1940 if $includeKraken and not KRAKEN.empty? 1941 $suites << KRAKEN 1942 end 1943 1944 $benchmarks = [] 1945 $suites.each { 1946 | suite | 1947 $benchmarks += suite.benchmarks 1948 } 1949 1950 $runPlans = [] 1951 $vms.each { 1952 | vm | 1953 $benchmarks.each { 1954 | benchmark | 1955 $outer.times { 1956 | iteration | 1957 $runPlans << BenchRunPlan.new(benchmark, vm, iteration) 1958 } 1959 } 1960 } 1961 1962 $runPlans.shuffle! 1963 1964 $suitepad = $suites.collect { 1965 | suite | 1966 suite.to_s.size 1967 }.max + 1 1968 1969 $benchpad = ($benchmarks + 1970 ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect { 1971 | benchmark | 1972 if benchmark.respond_to? :name 1973 benchmark.name.size 1974 else 1975 benchmark.size 1976 end 1977 }.max + 1 1978 1979 $vmpad = $vms.collect { 1980 | vm | 1981 vm.to_s.size 1982 }.max + 1 1983 1984 if $prepare 1985 File.open("#{BENCH_DATA_PATH}/runscript", "w") { 1986 | file | 1987 file.puts "echo \"HOSTNAME:\\c\"" 1988 file.puts "hostname" 1989 file.puts "echo" 1990 file.puts "echo \"HARDWARE:\\c\"" 1991 file.puts "/usr/sbin/sysctl hw.model" 1992 file.puts "echo" 1993 file.puts "set -e" 1994 $script = file 1995 $runPlans.each_with_index { 1996 | plan, idx | 1997 if $verbosity == 0 and not $silent 1998 text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s 1999 text2 = plan.benchmark.to_s+"/"+plan.vm.to_s 2000 file.puts("echo "+("\r#{text1} #{rpad(text2,$suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2") 2001 file.puts("echo "+("\r#{text1} #{text2}".inspect)[0..-2]+"\\c\" 1>&2") 2002 end 2003 plan.emitRunCode 2004 } 2005 if $verbosity == 0 and not $silent 2006 file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2") 2007 file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size}".inspect)+" 1>&2") 2008 end 2009 } 2010 end 2011 2012 if $run 2013 unless $remoteHosts.empty? 2014 $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0 2015 Dir.chdir(TEMP_PATH) { 2016 cmd = "tar -czf payload.tar.gz benchdata" 2017 $stderr.puts ">> #{cmd}" if $verbosity>=2 2018 raise unless system(cmd) 2019 } 2020 2021 def grokHost(host) 2022 if host =~ /:([0-9]+)$/ 2023 "-p " + $1 + " " + $~.pre_match.inspect 2024 else 2025 host.inspect 2026 end 2027 end 2028 2029 def sshRead(host, command) 2030 cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}" 2031 $stderr.puts ">> #{cmd}" if $verbosity>=2 2032 result = "" 2033 IO.popen(cmd, "r") { 2034 | inp | 2035 inp.each_line { 2036 | line | 2037 $stderr.puts "#{host}: #{line}" if $verbosity>=2 2038 result += line 2039 } 2040 } 2041 raise "#{$?}" unless $?.success? 2042 result 2043 end 2044 2045 def sshWrite(host, command, data) 2046 cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}" 2047 $stderr.puts ">> #{cmd}" if $verbosity>=2 2048 IO.popen(cmd, "w") { 2049 | outp | 2050 outp.write(data) 2051 } 2052 raise "#{$?}" unless $?.success? 2053 end 2054 2055 $remoteHosts.each { 2056 | host | 2057 $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0 2058 2059 remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"] 2060 raise unless remoteTempPath 2061 2062 sshWrite(host, "cd #{remoteTempPath.inspect} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz")) 2063 2064 $stderr.puts "Running on #{host}..." if $verbosity==0 2065 2066 parseAndDisplayResults(sshRead(host, "cd #{(remoteTempPath+'/benchdata').inspect} && sh runscript")) 2067 } 2068 end 2069 2070 if not $remoteHosts.empty? and $alsoLocal 2071 $stderr.puts "Running locally..." 2072 end 2073 2074 if $remoteHosts.empty? or $alsoLocal 2075 parseAndDisplayResults(runAndGetResults) 2076 end 2077 end 2078 2079 $analyze.each_with_index { 2080 | filename, index | 2081 if index >= 1 2082 puts 2083 end 2084 parseAndDisplayResults(IO::read(filename)) 2085 } 2086 2087 if $prepare and not $run and $analyze.empty? 2088 puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+ 2089 "the benchmarks and get the results by doing:", 78) 2090 puts 2091 puts "cd #{BENCH_DATA_PATH}" 2092 puts "sh runscript > results.txt" 2093 puts 2094 puts wrap("Then you can analyze the results by running bencher with the same arguments "+ 2095 "as now, but replacing --prepare-only with --analyze results.txt.", 78) 2096 end 2097 rescue => e 2098 fail(e) 2099 end 2100 2101 2102
