1 #!/usr/bin/env ruby 2 3 # Copyright (C) 2011 Apple Inc. All rights reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions 7 # are met: 8 # 1. Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # 2. Redistributions in binary form must reproduce the above copyright 11 # notice, this list of conditions and the following disclaimer in the 12 # documentation and/or other materials provided with the distribution. 13 # 14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' 15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS 18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 24 # THE POSSIBILITY OF SUCH DAMAGE. 25 26 require 'rubygems' 27 28 require 'getoptlong' 29 require 'pathname' 30 require 'tempfile' 31 require 'socket' 32 33 begin 34 require 'json' 35 rescue LoadError => e 36 $stderr.puts "It does not appear that you have the 'json' package installed. Try running 'sudo gem install json'." 37 exit 1 38 end 39 40 # Configuration 41 42 CONFIGURATION_FLNM = ENV["HOME"]+"/.bencher" 43 44 unless FileTest.exist? CONFIGURATION_FLNM 45 $stderr.puts "Error: no configuration file at ~/.bencher." 46 $stderr.puts "This file should contain paths to SunSpider, V8, and Kraken, as well as a" 47 $stderr.puts "temporary directory that bencher can use for its remote mode. It should be" 48 $stderr.puts "formatted in JSON. For example:" 49 $stderr.puts "{" 50 $stderr.puts " \"sunSpiderPath\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/sunspider-1.0\"," 51 $stderr.puts " \"v8Path\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/v8-v6\"," 52 $stderr.puts " \"krakenPath\": \"/Volumes/Data/pizlo/kraken/kraken-e119421cb325/tests/kraken-1.1\"," 53 $stderr.puts " \"tempPath\": \"/Volumes/Data/pizlo/bencher/temp\"" 54 $stderr.puts "}" 55 exit 1 56 end 57 58 CONFIGURATION = JSON.parse(File::read(CONFIGURATION_FLNM)) 59 60 SUNSPIDER_PATH = CONFIGURATION["sunSpiderPath"] 61 V8_PATH = CONFIGURATION["v8Path"] 62 KRAKEN_PATH = CONFIGURATION["krakenPath"] 63 TEMP_PATH = CONFIGURATION["tempPath"] 64 BENCH_DATA_PATH = TEMP_PATH + "/benchdata" 65 66 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 67 0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 68 0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 69 0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 70 0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 71 0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 72 0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 73 0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 74 0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 75 0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 76 0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 77 0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 78 0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 79 0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 80 0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 81 0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 82 0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 83 0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 84 0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 85 0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 86 0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 87 0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 88 0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 89 0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 90 0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 91 0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 92 0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 93 0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 94 0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 95 0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 96 0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 97 0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 98 0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 99 0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 100 0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 101 0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 102 0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 103 0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 104 0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 105 0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 106 0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 107 0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 108 0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 109 0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 110 0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 111 0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 112 0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 113 0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 114 0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 115 0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 116 0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 117 0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 118 0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 119 0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 120 0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 121 0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 122 0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 123 0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 124 0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 125 0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 126 0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 127 0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 128 0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 129 0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 130 0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 131 0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 132 0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 133 0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 134 0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 135 0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 136 0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 137 0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 138 0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 139 0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 140 0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 141 0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 142 0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 143 0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 144 0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 145 0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 146 0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 147 0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 148 0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 149 0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 150 0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 151 0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 152 0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 153 0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 154 0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 155 0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 156 0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 157 0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 158 0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 159 0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 160 0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 161 0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 162 0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 163 0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 164 0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 165 0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 166 0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 167 0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 168 0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 169 0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 170 0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 171 0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 172 0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 173 0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 174 0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 175 0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 176 0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 177 0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 178 0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 179 0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 180 0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 181 0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 182 0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 183 0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 184 0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 185 0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 186 0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 187 0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 188 0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 189 0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 190 0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 191 0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 192 0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 193 0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 194 0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 195 0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 196 0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 197 0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 198 0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 199 0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 200 0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 201 0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 202 0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 203 0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 204 0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 205 0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 206 0.99616, 0.996164] 207 208 # Run-time configuration parameters (can be set with command-line options) 209 210 $rerun=1 211 $inner=3 212 $warmup=1 213 $outer=4 214 $includeSunSpider=true 215 $includeV8=true 216 $includeKraken=true 217 $measureGC=false 218 $benchmarkPattern=nil 219 $verbosity=0 220 $timeMode=:preciseTime 221 $forceVMKind=nil 222 $brief=false 223 $silent=false 224 $remoteHosts=[] 225 $alsoLocal=false 226 $sshOptions=[] 227 $vms = [] 228 $needToCopyVMs = false 229 $dontCopyVMs = false 230 231 $prepare = true 232 $run = true 233 $analyze = [] 234 235 # Helpful functions and classes 236 237 def smallUsage 238 puts "Use the --help option to get basic usage information." 239 exit 1 240 end 241 242 def usage 243 puts "bencher [options] <vm1> [<vm2> ...]" 244 puts 245 puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken" 246 puts "benchmarks, and reports detailed statistics. What makes bencher special is" 247 puts "that each benchmark/VM configuration is run in a single VM invocation, and" 248 puts "the invocations are run in random order. This minimizes systematics due to" 249 puts "one benchmark polluting the running time of another. The fine-grained" 250 puts "interleaving of VM invocations further minimizes systematics due to changes in" 251 puts "the performance or behavior of your machine." 252 puts 253 puts "Bencher is highly configurable. You can compare as many VMs as you like. You" 254 puts "can change the amount of warm-up iterations, number of iterations executed per" 255 puts "VM invocation, and the number of VM invocations per benchmark. By default," 256 puts "SunSpider, VM, and Kraken are all run; but you can run any combination of these" 257 puts "suites." 258 puts 259 puts "The <vm> should be either a path to a JavaScript runtime executable (such as" 260 puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to" 261 puts "the executable and <name> is the name that you would like to give the" 262 puts "configuration for the purposeof reporting. If no name is given, a generic name" 263 puts "of the form Conf#<n> will be ascribed to the configuration automatically." 264 puts 265 puts "Options:" 266 puts "--rerun <n> Set the number of iterations of the benchmark that" 267 puts " contribute to the measured run time. Default is #{$rerun}." 268 puts "--inner <n> Set the number of inner (per-runtime-invocation)" 269 puts " iterations. Default is #{$inner}." 270 puts "--outer <n> Set the number of runtime invocations for each benchmark." 271 puts " Default is #{$outer}." 272 puts "--warmup <n> Set the number of warm-up runs per invocation. Default" 273 puts " is #{$warmup}." 274 puts "--timing-mode Set the way that bencher measures time. Possible values" 275 puts " are 'preciseTime' and 'date'. Default is 'preciseTime'." 276 puts "--force-vm-kind Turn off auto-detection of VM kind, and assume that it is" 277 puts " the one specified. Valid arguments are 'jsc' or" 278 puts " 'DumpRenderTree'." 279 puts "--force-vm-copy Force VM builds to be copied to bencher's working directory." 280 puts " This may reduce pathologies resulting from path names." 281 puts "--dont-copy-vms Don't copy VMs even when doing a remote benchmarking run;" 282 puts " instead assume that they are already there." 283 puts "--v8-only Only run V8." 284 puts "--sunspider-only Only run SunSpider." 285 puts "--kraken-only Only run Kraken." 286 puts "--exclude-v8 Exclude V8 (only run SunSpider and Kraken)." 287 puts "--exclude-sunspider Exclude SunSpider (only run V8 and Kraken)." 288 puts "--exclude-kraken Exclude Kraken (only run SunSpider and V8)." 289 puts "--benchmarks Only run benchmarks matching the given regular expression." 290 puts "--measure-gc Turn off manual calls to gc(), so that GC time is measured." 291 puts " Works best with large values of --inner. You can also say" 292 puts " --measure-gc <conf>, which turns this on for one" 293 puts " configuration only." 294 puts "--verbose or -v Print more stuff." 295 puts "--brief Print only the final result for each VM." 296 puts "--silent Don't print progress. This might slightly reduce some" 297 puts " performance perturbation." 298 puts "--remote <sshhosts> Performance performance measurements remotely, on the given" 299 puts " SSH host(s). Easiest way to use this is to specify the SSH" 300 puts " user@host string. However, you can also supply a comma-" 301 puts " separated list of SSH hosts. Alternatively, you can use this" 302 puts " option multiple times to specify multiple hosts. This" 303 puts " automatically copies the WebKit release builds of the VMs" 304 puts " you specified to all of the hosts." 305 puts "--ssh-options Pass additional options to SSH." 306 puts "--local Also do a local benchmark run even when doing --remote." 307 puts "--prepare-only Only prepare the bencher runscript (a shell script that" 308 puts " invokes the VMs to run benchmarks) but don't run it." 309 puts "--analyze Only read the output of the runscript but don't do anything" 310 puts " else. This requires passing the same arguments to bencher" 311 puts " that you passed when running --prepare-only." 312 puts "--help or -h Display this message." 313 puts 314 puts "Example:" 315 puts "bencher TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc" 316 exit 1 317 end 318 319 def fail(reason) 320 if reason.respond_to? :backtrace 321 puts "FAILED: #{reason}" 322 puts "Stack trace:" 323 puts reason.backtrace.join("\n") 324 else 325 puts "FAILED: #{reason}" 326 end 327 smallUsage 328 end 329 330 def quickFail(r1,r2) 331 $stderr.puts "#{$0}: #{r1}" 332 puts 333 fail(r2) 334 end 335 336 def intArg(argName,arg,min,max) 337 result=arg.to_i 338 unless result.to_s == arg 339 quickFail("Expected an integer value for #{argName}, but got #{arg}.", 340 "Invalid argument for command-line option") 341 end 342 if min and result<min 343 quickFail("Argument for #{argName} cannot be smaller than #{min}.", 344 "Invalid argument for command-line option") 345 end 346 if max and result>max 347 quickFail("Argument for #{argName} cannot be greater than #{max}.", 348 "Invalid argument for command-line option") 349 end 350 result 351 end 352 353 def computeMean(array) 354 sum=0.0 355 array.each { 356 | value | 357 sum += value 358 } 359 sum/array.length 360 end 361 362 def computeGeometricMean(array) 363 mult=1.0 364 array.each { 365 | value | 366 mult*=value 367 } 368 mult**(1.0/array.length) 369 end 370 371 def computeHarmonicMean(array) 372 1.0 / computeMean(array.collect{ | value | 1.0 / value }) 373 end 374 375 def computeStdDev(array) 376 case array.length 377 when 0 378 0.0/0.0 379 when 1 380 0.0 381 else 382 begin 383 mean=computeMean(array) 384 sum=0.0 385 array.each { 386 | value | 387 sum += (value-mean)**2 388 } 389 Math.sqrt(sum/(array.length-1)) 390 rescue 391 0.0/0.0 392 end 393 end 394 end 395 396 class Array 397 def shuffle! 398 size.downto(1) { |n| push delete_at(rand(n)) } 399 self 400 end 401 end 402 403 def inverseBetaRegularized(n) 404 IBR_LOOKUP[n-1] 405 end 406 407 def numToStr(num) 408 "%.4f"%(num.to_f) 409 end 410 411 class NoChange 412 attr_reader :amountFaster 413 414 def initialize(amountFaster) 415 @amountFaster = amountFaster 416 end 417 418 def shortForm 419 " " 420 end 421 422 def longForm 423 " might be #{numToStr(@amountFaster)}x faster" 424 end 425 426 def to_s 427 if @amountFaster < 1.01 428 "" 429 else 430 longForm 431 end 432 end 433 end 434 435 class Faster 436 attr_reader :amountFaster 437 438 def initialize(amountFaster) 439 @amountFaster = amountFaster 440 end 441 442 def shortForm 443 "^" 444 end 445 446 def longForm 447 "^ definitely #{numToStr(@amountFaster)}x faster" 448 end 449 450 def to_s 451 longForm 452 end 453 end 454 455 class Slower 456 attr_reader :amountSlower 457 458 def initialize(amountSlower) 459 @amountSlower = amountSlower 460 end 461 462 def shortForm 463 "!" 464 end 465 466 def longForm 467 "! definitely #{numToStr(@amountSlower)}x slower" 468 end 469 470 def to_s 471 longForm 472 end 473 end 474 475 class MayBeSlower 476 attr_reader :amountSlower 477 478 def initialize(amountSlower) 479 @amountSlower = amountSlower 480 end 481 482 def shortForm 483 "?" 484 end 485 486 def longForm 487 "? might be #{numToStr(@amountSlower)}x slower" 488 end 489 490 def to_s 491 if @amountSlower < 1.01 492 "?" 493 else 494 longForm 495 end 496 end 497 end 498 499 class Stats 500 def initialize 501 @array = [] 502 end 503 504 def add(value) 505 if value.is_a? Stats 506 add(value.array) 507 elsif value.respond_to? :each 508 value.each { 509 | v | 510 add(v) 511 } 512 else 513 @array << value.to_f 514 end 515 end 516 517 def array 518 @array 519 end 520 521 def sum 522 result=0 523 @array.each { 524 | value | 525 result += value 526 } 527 result 528 end 529 530 def min 531 @array.min 532 end 533 534 def max 535 @array.max 536 end 537 538 def size 539 @array.length 540 end 541 542 def mean 543 computeMean(array) 544 end 545 546 def arithmeticMean 547 mean 548 end 549 550 def stdDev 551 computeStdDev(array) 552 end 553 554 def stdErr 555 stdDev/Math.sqrt(size) 556 end 557 558 # Computes a 95% Student's t distribution confidence interval 559 def confInt 560 if size < 2 561 0.0/0.0 562 else 563 raise if size > 1000 564 Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1)) 565 end 566 end 567 568 def lower 569 mean-confInt 570 end 571 572 def upper 573 mean+confInt 574 end 575 576 def geometricMean 577 computeGeometricMean(array) 578 end 579 580 def harmonicMean 581 computeHarmonicMean(array) 582 end 583 584 def compareTo(other) 585 if upper < other.lower 586 Faster.new(other.mean/mean) 587 elsif lower > other.upper 588 Slower.new(mean/other.mean) 589 elsif mean > other.mean 590 MayBeSlower.new(mean/other.mean) 591 else 592 NoChange.new(other.mean/mean) 593 end 594 end 595 596 def to_s 597 "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}" 598 end 599 end 600 601 def doublePuts(out1,out2,msg) 602 out1.puts "#{out2.path}: #{msg}" if $verbosity>=3 603 out2.puts msg 604 end 605 606 class Benchfile < File 607 @@counter = 0 608 609 attr_reader :filename, :basename 610 611 def initialize(name) 612 @basename, @filename = Benchfile.uniqueFilename(name) 613 super(@filename, "w") 614 end 615 616 def self.uniqueFilename(name) 617 if name.is_a? Array 618 basename = name[0] + @@counter.to_s + name[1] 619 else 620 basename = name + @@counter.to_s 621 end 622 filename = BENCH_DATA_PATH + "/" + basename 623 @@counter += 1 624 raise "Benchfile #{filename} already exists" if FileTest.exist?(filename) 625 [basename, filename] 626 end 627 628 def self.create(name) 629 file = Benchfile.new(name) 630 yield file 631 file.close 632 file.basename 633 end 634 end 635 636 $dataFiles={} 637 def ensureFile(key, filename) 638 unless $dataFiles[key] 639 $dataFiles[key] = Benchfile.create(key) { 640 | outp | 641 doublePuts($stderr,outp,IO::read(filename)) 642 } 643 end 644 $dataFiles[key] 645 end 646 647 def emitBenchRunCodeFile(name, plan, benchDataPath, benchPath) 648 case plan.vm.vmType 649 when :jsc 650 Benchfile.create("bencher") { 651 | file | 652 case $timeMode 653 when :preciseTime 654 doublePuts($stderr,file,"function __bencher_curTimeMS() {") 655 doublePuts($stderr,file," return preciseTime()*1000") 656 doublePuts($stderr,file,"}") 657 when :date 658 doublePuts($stderr,file,"function __bencher_curTimeMS() {") 659 doublePuts($stderr,file," return Date.now()") 660 doublePuts($stderr,file,"}") 661 else 662 raise 663 end 664 665 if benchDataPath 666 doublePuts($stderr,file,"load(#{benchDataPath.inspect});") 667 doublePuts($stderr,file,"gc();") 668 doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {") 669 doublePuts($stderr,file," before = __bencher_curTimeMS();") 670 $rerun.times { 671 doublePuts($stderr,file," load(#{benchPath.inspect});") 672 } 673 doublePuts($stderr,file," after = __bencher_curTimeMS();") 674 doublePuts($stderr,file," if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(after-before));"); 675 doublePuts($stderr,file," gc();") unless plan.vm.shouldMeasureGC 676 doublePuts($stderr,file,"}") 677 else 678 doublePuts($stderr,file,"function __bencher_run(__bencher_what) {") 679 doublePuts($stderr,file," var __bencher_before = __bencher_curTimeMS();") 680 $rerun.times { 681 doublePuts($stderr,file," run(__bencher_what);") 682 } 683 doublePuts($stderr,file," var __bencher_after = __bencher_curTimeMS();") 684 doublePuts($stderr,file," return __bencher_after - __bencher_before;") 685 doublePuts($stderr,file,"}") 686 $warmup.times { 687 doublePuts($stderr,file,"__bencher_run(#{benchPath.inspect})") 688 doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC 689 } 690 $inner.times { 691 | innerIndex | 692 doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchPath.inspect}));") 693 doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC 694 } 695 end 696 } 697 when :dumpRenderTree 698 mainCode = Benchfile.create("bencher") { 699 | file | 700 doublePuts($stderr,file,"__bencher_count = 0;") 701 doublePuts($stderr,file,"function __bencher_doNext(result) {") 702 doublePuts($stderr,file," if (__bencher_count >= #{$warmup})") 703 doublePuts($stderr,file," debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);") 704 doublePuts($stderr,file," __bencher_count++;") 705 doublePuts($stderr,file," if (__bencher_count < #{$inner+$warmup})") 706 doublePuts($stderr,file," __bencher_runImpl(__bencher_doNext);") 707 doublePuts($stderr,file," else") 708 doublePuts($stderr,file," quit();") 709 doublePuts($stderr,file,"}") 710 doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);") 711 } 712 713 cssCode = Benchfile.create("bencher-css") { 714 | file | 715 doublePuts($stderr,file,".pass {\n font-weight: bold;\n color: green;\n}\n.fail {\n font-weight: bold;\n color: red;\n}\n\#console {\n white-space: pre-wrap;\n font-family: monospace;\n}") 716 } 717 718 preCode = Benchfile.create("bencher-pre") { 719 | file | 720 doublePuts($stderr,file,"if (window.testRunner) {") 721 doublePuts($stderr,file," testRunner.dumpAsText(window.enablePixelTesting);") 722 doublePuts($stderr,file," testRunner.waitUntilDone();") 723 doublePuts($stderr,file,"}") 724 doublePuts($stderr,file,"") 725 doublePuts($stderr,file,"function debug(msg)") 726 doublePuts($stderr,file,"{") 727 doublePuts($stderr,file," var span = document.createElement(\"span\");") 728 doublePuts($stderr,file," document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace") 729 doublePuts($stderr,file," span.innerHTML = msg + '<br />';") 730 doublePuts($stderr,file,"}") 731 doublePuts($stderr,file,"") 732 doublePuts($stderr,file,"function quit() {") 733 doublePuts($stderr,file," testRunner.notifyDone();") 734 doublePuts($stderr,file,"}") 735 doublePuts($stderr,file,"") 736 doublePuts($stderr,file,"__bencher_continuation=null;") 737 doublePuts($stderr,file,"") 738 doublePuts($stderr,file,"function reportResult(result) {") 739 doublePuts($stderr,file," __bencher_continuation(result);") 740 doublePuts($stderr,file,"}") 741 doublePuts($stderr,file,"") 742 doublePuts($stderr,file,"function __bencher_runImpl(continuation) {") 743 doublePuts($stderr,file," function doit() {") 744 doublePuts($stderr,file," document.getElementById(\"frameparent\").innerHTML = \"\";") 745 doublePuts($stderr,file," document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";") 746 doublePuts($stderr,file," var testFrame = document.getElementById(\"testframe\");") 747 doublePuts($stderr,file," testFrame.contentDocument.open();") 748 doublePuts($stderr,file," testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");") 749 if benchDataPath 750 doublePuts($stderr,file," testFrame.contentDocument.write(\"<script src=\\\"#{benchDataPath}\\\"></script>\");") 751 end 752 doublePuts($stderr,file," testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">__bencher_before = Date.now();</script><script src=\\\"#{benchPath}\\\"></script><script type=\\\"text/javascript\\\">window.parent.reportResult(Date.now() - __bencher_before);</script></body></html>\");") 753 doublePuts($stderr,file," testFrame.contentDocument.close();") 754 doublePuts($stderr,file," }") 755 doublePuts($stderr,file," __bencher_continuation = continuation;") 756 doublePuts($stderr,file," window.setTimeout(doit, 10);") 757 doublePuts($stderr,file,"}") 758 } 759 760 Benchfile.create(["bencher-htmldoc",".html"]) { 761 | file | 762 doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>") 763 } 764 else 765 raise 766 end 767 end 768 769 def emitBenchRunCode(name, plan, benchDataPath, benchPath) 770 plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchDataPath, benchPath)) 771 end 772 773 def planForDescription(plans, benchFullname, vmName, iteration) 774 raise unless benchFullname =~ /\// 775 suiteName = $~.pre_match 776 benchName = $~.post_match 777 result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration} 778 raise unless result.size == 1 779 result[0] 780 end 781 782 class ParsedResult 783 attr_reader :plan, :innerIndex, :time 784 785 def initialize(plan, innerIndex, time) 786 @plan = plan 787 @innerIndex = innerIndex 788 @time = time 789 790 raise unless @plan.is_a? BenchPlan 791 raise unless @innerIndex.is_a? Integer 792 raise unless @time.is_a? Numeric 793 end 794 795 def benchmark 796 plan.benchmark 797 end 798 799 def suite 800 plan.suite 801 end 802 803 def vm 804 plan.vm 805 end 806 807 def outerIndex 808 plan.iteration 809 end 810 811 def self.parse(plans, string) 812 if string =~ /([a-zA-Z0-9\/-]+): ([a-zA-Z0-9_# ]+): ([0-9]+): ([0-9]+): Time: / 813 benchFullname = $1 814 vmName = $2 815 outerIndex = $3.to_i 816 innerIndex = $4.to_i 817 time = $~.post_match.to_f 818 ParsedResult.new(planForDescription(plans, benchFullname, vmName, outerIndex), innerIndex, time) 819 else 820 nil 821 end 822 end 823 end 824 825 class VM 826 def initialize(origPath, name, nameKind, svnRevision) 827 @origPath = origPath.to_s 828 @path = origPath.to_s 829 @name = name 830 @nameKind = nameKind 831 832 if $forceVMKind 833 @vmType = $forceVMKind 834 else 835 if @origPath =~ /DumpRenderTree$/ 836 @vmType = :dumpRenderTree 837 else 838 @vmType = :jsc 839 end 840 end 841 842 @svnRevision = svnRevision 843 844 # Try to detect information about the VM. 845 if path =~ /\/WebKitBuild\/Release\/([a-zA-Z]+)$/ 846 @checkoutPath = $~.pre_match 847 # FIXME: Use some variant of this: 848 # <bdash> def retrieve_revision 849 # <bdash> `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i 850 # <bdash> end 851 unless @svnRevision 852 begin 853 Dir.chdir(@checkoutPath) { 854 $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2 855 IO.popen("svn info", "r") { 856 | inp | 857 inp.each_line { 858 | line | 859 if line =~ /Revision: ([0-9]+)/ 860 @svnRevision = $1 861 end 862 } 863 } 864 } 865 unless @svnRevision 866 $stderr.puts "Warning: running svn info for #{name} silently failed." 867 end 868 rescue => e 869 # Failed to detect svn revision. 870 $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}" 871 end 872 end 873 else 874 $stderr.puts "Warning: could not identify checkout location for #{name}" 875 end 876 877 if @path =~ /\/Release\/([a-zA-Z]+)$/ 878 @libPath, @relativeBinPath = $~.pre_match+"/Release", "./#{$1}" 879 elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/ 880 @libPath = $~.pre_match 881 elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/ 882 @libPath, @relativeBinPath = $~.pre_match, $&[1..-1] 883 end 884 end 885 886 def canCopyIntoBenchPath 887 if @libPath and @relativeBinPath 888 true 889 else 890 false 891 end 892 end 893 894 def copyIntoBenchPath 895 raise unless canCopyIntoBenchPath 896 basename, filename = Benchfile.uniqueFilename("vm") 897 raise unless Dir.mkdir(filename) 898 cmd = "cp -a #{@libPath.inspect}/* #{filename.inspect}" 899 $stderr.puts ">> #{cmd}" if $verbosity>=2 900 raise unless system(cmd) 901 @path = "#{basename}/#{@relativeBinPath}" 902 @libPath = basename 903 end 904 905 def to_s 906 @name 907 end 908 909 def name 910 @name 911 end 912 913 def shouldMeasureGC 914 $measureGC == true or ($measureGC == name) 915 end 916 917 def origPath 918 @origPath 919 end 920 921 def path 922 @path 923 end 924 925 def nameKind 926 @nameKind 927 end 928 929 def vmType 930 @vmType 931 end 932 933 def checkoutPath 934 @checkoutPath 935 end 936 937 def svnRevision 938 @svnRevision 939 end 940 941 def printFunction 942 case @vmType 943 when :jsc 944 "print" 945 when :dumpRenderTree 946 "debug" 947 else 948 raise @vmType 949 end 950 end 951 952 def emitRunCode(fileToRun) 953 myLibPath = @libPath 954 myLibPath = "" unless myLibPath 955 $script.puts "export DYLD_LIBRARY_PATH=#{myLibPath.to_s.inspect}" 956 $script.puts "export DYLD_FRAMEWORK_PATH=#{myLibPath.to_s.inspect}" 957 $script.puts "#{path} #{fileToRun}" 958 end 959 end 960 961 class StatsAccumulator 962 def initialize 963 @stats = [] 964 ($outer*$inner).times { 965 @stats << Stats.new 966 } 967 end 968 969 def statsForIteration(outerIteration, innerIteration) 970 @stats[outerIteration*$inner + innerIteration] 971 end 972 973 def stats 974 result = Stats.new 975 @stats.each { 976 | stat | 977 result.add(yield stat) 978 } 979 result 980 end 981 982 def geometricMeanStats 983 stats { 984 | stat | 985 stat.geometricMean 986 } 987 end 988 989 def arithmeticMeanStats 990 stats { 991 | stat | 992 stat.arithmeticMean 993 } 994 end 995 end 996 997 module Benchmark 998 attr_accessor :benchmarkSuite 999 attr_reader :name 1000 1001 def fullname 1002 benchmarkSuite.name + "/" + name 1003 end 1004 1005 def to_s 1006 fullname 1007 end 1008 end 1009 1010 class SunSpiderBenchmark 1011 include Benchmark 1012 1013 def initialize(name) 1014 @name = name 1015 end 1016 1017 def emitRunCode(plan) 1018 emitBenchRunCode(fullname, plan, nil, ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")) 1019 end 1020 end 1021 1022 class V8Benchmark 1023 include Benchmark 1024 1025 def initialize(name) 1026 @name = name 1027 end 1028 1029 def emitRunCode(plan) 1030 emitBenchRunCode(fullname, plan, nil, ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")) 1031 end 1032 end 1033 1034 class KrakenBenchmark 1035 include Benchmark 1036 1037 def initialize(name) 1038 @name = name 1039 end 1040 1041 def emitRunCode(plan) 1042 emitBenchRunCode(fullname, plan, ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js"), ensureFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")) 1043 end 1044 end 1045 1046 class BenchmarkSuite 1047 def initialize(name, path, preferredMean) 1048 @name = name 1049 @path = path 1050 @preferredMean = preferredMean 1051 @benchmarks = [] 1052 end 1053 1054 def name 1055 @name 1056 end 1057 1058 def to_s 1059 @name 1060 end 1061 1062 def path 1063 @path 1064 end 1065 1066 def add(benchmark) 1067 if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern 1068 benchmark.benchmarkSuite = self 1069 @benchmarks << benchmark 1070 end 1071 end 1072 1073 def benchmarks 1074 @benchmarks 1075 end 1076 1077 def benchmarkForName(name) 1078 result = @benchmarks.select{|v| v.name == name} 1079 raise unless result.length == 1 1080 result[0] 1081 end 1082 1083 def empty? 1084 @benchmarks.empty? 1085 end 1086 1087 def retain_if 1088 @benchmarks.delete_if { 1089 | benchmark | 1090 not yield benchmark 1091 } 1092 end 1093 1094 def preferredMean 1095 @preferredMean 1096 end 1097 1098 def computeMean(stat) 1099 stat.send @preferredMean 1100 end 1101 end 1102 1103 class BenchRunPlan 1104 def initialize(benchmark, vm, iteration) 1105 @benchmark = benchmark 1106 @vm = vm 1107 @iteration = iteration 1108 end 1109 1110 def benchmark 1111 @benchmark 1112 end 1113 1114 def suite 1115 @benchmark.benchmarkSuite 1116 end 1117 1118 def vm 1119 @vm 1120 end 1121 1122 def iteration 1123 @iteration 1124 end 1125 1126 def emitRunCode 1127 @benchmark.emitRunCode(self) 1128 end 1129 end 1130 1131 class BenchmarkOnVM 1132 def initialize(benchmark, suiteOnVM) 1133 @benchmark = benchmark 1134 @suiteOnVM = suiteOnVM 1135 @stats = Stats.new 1136 end 1137 1138 def to_s 1139 "#{@benchmark} on #{@suiteOnVM.vm}" 1140 end 1141 1142 def benchmark 1143 @benchmark 1144 end 1145 1146 def vm 1147 @suiteOnVM.vm 1148 end 1149 1150 def vmStats 1151 @suiteOnVM.vmStats 1152 end 1153 1154 def suite 1155 @benchmark.benchmarkSuite 1156 end 1157 1158 def suiteOnVM 1159 @suiteOnVM 1160 end 1161 1162 def stats 1163 @stats 1164 end 1165 1166 def parseResult(result) 1167 raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm 1168 raise unless result.benchmark == @benchmark 1169 @stats.add(result.time) 1170 end 1171 end 1172 1173 class SuiteOnVM < StatsAccumulator 1174 def initialize(vm, vmStats, suite) 1175 super() 1176 @vm = vm 1177 @vmStats = vmStats 1178 @suite = suite 1179 1180 raise unless @vm.is_a? VM 1181 raise unless @vmStats.is_a? StatsAccumulator 1182 raise unless @suite.is_a? BenchmarkSuite 1183 end 1184 1185 def to_s 1186 "#{@suite} on #{@vm}" 1187 end 1188 1189 def suite 1190 @suite 1191 end 1192 1193 def vm 1194 @vm 1195 end 1196 1197 def vmStats 1198 raise unless @vmStats 1199 @vmStats 1200 end 1201 end 1202 1203 class BenchPlan 1204 def initialize(benchmarkOnVM, iteration) 1205 @benchmarkOnVM = benchmarkOnVM 1206 @iteration = iteration 1207 end 1208 1209 def to_s 1210 "#{@benchmarkOnVM} \##{@iteration+1}" 1211 end 1212 1213 def benchmarkOnVM 1214 @benchmarkOnVM 1215 end 1216 1217 def benchmark 1218 @benchmarkOnVM.benchmark 1219 end 1220 1221 def suite 1222 @benchmarkOnVM.suite 1223 end 1224 1225 def vm 1226 @benchmarkOnVM.vm 1227 end 1228 1229 def iteration 1230 @iteration 1231 end 1232 1233 def parseResult(result) 1234 raise unless result.plan == self 1235 @benchmarkOnVM.parseResult(result) 1236 @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time) 1237 @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time) 1238 end 1239 end 1240 1241 def lpad(str,chars) 1242 if str.length>chars 1243 str 1244 else 1245 "%#{chars}s"%(str) 1246 end 1247 end 1248 1249 def rpad(str,chars) 1250 while str.length<chars 1251 str+=" " 1252 end 1253 str 1254 end 1255 1256 def center(str,chars) 1257 while str.length<chars 1258 str+=" " 1259 if str.length<chars 1260 str=" "+str 1261 end 1262 end 1263 str 1264 end 1265 1266 def statsToStr(stats) 1267 if $inner*$outer == 1 1268 string = numToStr(stats.mean) 1269 raise unless string =~ /\./ 1270 left = $~.pre_match 1271 right = $~.post_match 1272 lpad(left,12)+"."+rpad(right,9) 1273 else 1274 lpad(numToStr(stats.mean),11)+"+-"+rpad(numToStr(stats.confInt),9) 1275 end 1276 end 1277 1278 def plural(num) 1279 if num == 1 1280 "" 1281 else 1282 "s" 1283 end 1284 end 1285 1286 def wrap(str, columns) 1287 array = str.split 1288 result = "" 1289 curLine = array.shift 1290 array.each { 1291 | curStr | 1292 if (curLine + " " + curStr).size > columns 1293 result += curLine + "\n" 1294 curLine = curStr 1295 else 1296 curLine += " " + curStr 1297 end 1298 } 1299 result + curLine + "\n" 1300 end 1301 1302 def runAndGetResults 1303 results = nil 1304 Dir.chdir(BENCH_DATA_PATH) { 1305 IO.popen("sh ./runscript", "r") { 1306 | inp | 1307 results = inp.read 1308 } 1309 raise "Script did not complete correctly: #{$?}" unless $?.success? 1310 } 1311 raise unless results 1312 results 1313 end 1314 1315 def parseAndDisplayResults(results) 1316 vmStatses = [] 1317 $vms.each { 1318 vmStatses << StatsAccumulator.new 1319 } 1320 1321 suitesOnVMs = [] 1322 suitesOnVMsForSuite = {} 1323 $suites.each { 1324 | suite | 1325 suitesOnVMsForSuite[suite] = [] 1326 } 1327 suitesOnVMsForVM = {} 1328 $vms.each { 1329 | vm | 1330 suitesOnVMsForVM[vm] = [] 1331 } 1332 1333 benchmarksOnVMs = [] 1334 benchmarksOnVMsForBenchmark = {} 1335 $benchmarks.each { 1336 | benchmark | 1337 benchmarksOnVMsForBenchmark[benchmark] = [] 1338 } 1339 1340 $vms.each_with_index { 1341 | vm, vmIndex | 1342 vmStats = vmStatses[vmIndex] 1343 $suites.each { 1344 | suite | 1345 suiteOnVM = SuiteOnVM.new(vm, vmStats, suite) 1346 suitesOnVMs << suiteOnVM 1347 suitesOnVMsForSuite[suite] << suiteOnVM 1348 suitesOnVMsForVM[vm] << suiteOnVM 1349 suite.benchmarks.each { 1350 | benchmark | 1351 benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM) 1352 benchmarksOnVMs << benchmarkOnVM 1353 benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM 1354 } 1355 } 1356 } 1357 1358 plans = [] 1359 benchmarksOnVMs.each { 1360 | benchmarkOnVM | 1361 $outer.times { 1362 | iteration | 1363 plans << BenchPlan.new(benchmarkOnVM, iteration) 1364 } 1365 } 1366 1367 hostname = nil 1368 hwmodel = nil 1369 results.each_line { 1370 | line | 1371 line.chomp! 1372 if line =~ /HOSTNAME:([^.]+)/ 1373 hostname = $1 1374 elsif line =~ /HARDWARE:hw\.model: / 1375 hwmodel = $~.post_match.chomp 1376 else 1377 result = ParsedResult.parse(plans, line.chomp) 1378 if result 1379 result.plan.parseResult(result) 1380 end 1381 end 1382 } 1383 1384 # Compute the geomean of the preferred means of results on a SuiteOnVM 1385 overallResults = [] 1386 $vms.each { 1387 | vm | 1388 result = Stats.new 1389 $outer.times { 1390 | outerIndex | 1391 $inner.times { 1392 | innerIndex | 1393 curResult = Stats.new 1394 suitesOnVMsForVM[vm].each { 1395 | suiteOnVM | 1396 # For a given iteration, suite, and VM, compute the suite's preferred mean 1397 # over the data collected for all benchmarks in that suite. We'll have one 1398 # sample per benchmark. For example on V8 this will be the geomean of 1 1399 # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for 1400 # splay. 1401 curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex))) 1402 } 1403 1404 # curResult now holds 1 sample for each of the means computed in the above 1405 # loop. Compute the geomean over this, and store it. 1406 result.add(curResult.geometricMean) 1407 } 1408 } 1409 1410 # $overallResults will have a Stats for each VM. That Stats object will hold 1411 # $inner*$outer geomeans, allowing us to compute the arithmetic mean and 1412 # confidence interval of the geomeans of preferred means. Convoluted, but 1413 # useful and probably sound. 1414 overallResults << result 1415 } 1416 1417 if $verbosity >= 2 1418 benchmarksOnVMs.each { 1419 | benchmarkOnVM | 1420 $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}" 1421 } 1422 1423 $vms.each_with_index { 1424 | vm, vmIndex | 1425 vmStats = vmStatses[vmIndex] 1426 $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}" 1427 $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}" 1428 } 1429 end 1430 1431 reportName = 1432 (if ($vms.collect { 1433 | vm | 1434 vm.nameKind 1435 }.index :auto) 1436 "" 1437 else 1438 $vms.collect { 1439 | vm | 1440 vm.to_s 1441 }.join("_") + "_" 1442 end) + 1443 ($suites.collect { 1444 | suite | 1445 suite.to_s 1446 }.join("")) + "_" + 1447 (if hostname 1448 hostname + "_" 1449 else 1450 "" 1451 end)+ 1452 (begin 1453 time = Time.now 1454 "%04d%02d%02d_%02d%02d" % 1455 [ time.year, time.month, time.day, 1456 time.hour, time.min ] 1457 end) + 1458 "_benchReport.txt" 1459 1460 unless $brief 1461 puts "Generating benchmark report at #{reportName}" 1462 end 1463 1464 outp = $stdout 1465 begin 1466 outp = File.open(reportName,"w") 1467 rescue => e 1468 $stderr.puts "Error: could not save report to #{reportName}: #{e}" 1469 $stderr.puts 1470 end 1471 1472 def createVMsString 1473 result = "" 1474 result += " " if $suites.size > 1 1475 result += rpad("", $benchpad) 1476 result += " " 1477 $vms.size.times { 1478 | index | 1479 if index != 0 1480 result += " "+NoChange.new(0).shortForm 1481 end 1482 result += lpad(center($vms[index].name, 9+9+2), 11+9+2) 1483 } 1484 result += " " 1485 if $vms.size >= 3 1486 result += center("#{$vms[-1].name} v. #{$vms[0].name}",26) 1487 elsif $vms.size >= 2 1488 result += " "*26 1489 end 1490 result 1491 end 1492 1493 columns = [createVMsString.size, 78].max 1494 1495 outp.print "Benchmark report for " 1496 if $suites.size == 1 1497 outp.print $suites[0].to_s 1498 elsif $suites.size == 2 1499 outp.print "#{$suites[0]} and #{$suites[1]}" 1500 else 1501 outp.print "#{$suites[0..-2].join(', ')}, and #{$suites[-1]}" 1502 end 1503 if hostname 1504 outp.print " on #{hostname}" 1505 end 1506 if hwmodel 1507 outp.print " (#{hwmodel})" 1508 end 1509 outp.puts "." 1510 outp.puts 1511 1512 # This looks stupid; revisit later. 1513 if false 1514 $suites.each { 1515 | suite | 1516 outp.puts "#{suite} at #{suite.path}" 1517 } 1518 1519 outp.puts 1520 end 1521 1522 outp.puts "VMs tested:" 1523 $vms.each { 1524 | vm | 1525 outp.print "\"#{vm.name}\" at #{vm.origPath}" 1526 if vm.svnRevision 1527 outp.print " (r#{vm.svnRevision})" 1528 end 1529 outp.puts 1530 } 1531 1532 outp.puts 1533 1534 outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+ 1535 "with #{$outer} VM invocation#{plural($outer)} per benchmark."+ 1536 (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+ 1537 "total time of those iterations, for each sample.") 1538 else "" end)+ 1539 (if $measureGC == true then (" No manual garbage collection invocations were "+ 1540 "emitted.") 1541 elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+ 1542 "all VMs except #{$measureGC}.") 1543 else (" Emitted a call to gc() between sample measurements.") end)+ 1544 (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+ 1545 "began with the very first iteration.") 1546 else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+ 1547 "invocation for warm-up.") end)+ 1548 (case $timeMode 1549 when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+ 1550 "microsecond-level timing.") 1551 when :date then (" Used the portable Date.now() method to get millisecond-"+ 1552 "level timing.") 1553 else raise end)+ 1554 " Reporting benchmark execution times with 95% confidence "+ 1555 "intervals in milliseconds.", 1556 columns) 1557 1558 outp.puts 1559 1560 def printVMs(outp) 1561 outp.puts createVMsString 1562 end 1563 1564 def summaryStats(outp, accumulators, name, &proc) 1565 outp.print " " if $suites.size > 1 1566 outp.print rpad(name, $benchpad) 1567 outp.print " " 1568 accumulators.size.times { 1569 | index | 1570 if index != 0 1571 outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm 1572 end 1573 outp.print statsToStr(accumulators[index].stats(&proc)) 1574 } 1575 if accumulators.size>=2 1576 outp.print(" "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm) 1577 end 1578 outp.puts 1579 end 1580 1581 def meanName(currentMean, preferredMean) 1582 result = "<#{currentMean}>" 1583 if "#{currentMean}Mean" == preferredMean.to_s 1584 result += " *" 1585 end 1586 result 1587 end 1588 1589 def allSummaryStats(outp, accumulators, preferredMean) 1590 summaryStats(outp, accumulators, meanName("arithmetic", preferredMean)) { 1591 | stat | 1592 stat.arithmeticMean 1593 } 1594 1595 summaryStats(outp, accumulators, meanName("geometric", preferredMean)) { 1596 | stat | 1597 stat.geometricMean 1598 } 1599 1600 summaryStats(outp, accumulators, meanName("harmonic", preferredMean)) { 1601 | stat | 1602 stat.harmonicMean 1603 } 1604 end 1605 1606 $suites.each { 1607 | suite | 1608 printVMs(outp) 1609 if $suites.size > 1 1610 outp.puts "#{suite.name}:" 1611 else 1612 outp.puts 1613 end 1614 suite.benchmarks.each { 1615 | benchmark | 1616 outp.print " " if $suites.size > 1 1617 outp.print rpad(benchmark.name, $benchpad) 1618 outp.print " " 1619 myConfigs = benchmarksOnVMsForBenchmark[benchmark] 1620 myConfigs.size.times { 1621 | index | 1622 if index != 0 1623 outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm 1624 end 1625 outp.print statsToStr(myConfigs[index].stats) 1626 } 1627 if $vms.size>=2 1628 outp.print(" "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s) 1629 end 1630 outp.puts 1631 } 1632 outp.puts 1633 allSummaryStats(outp, suitesOnVMsForSuite[suite], suite.preferredMean) 1634 outp.puts if $suites.size > 1 1635 } 1636 1637 if $suites.size > 1 1638 printVMs(outp) 1639 outp.puts "All benchmarks:" 1640 allSummaryStats(outp, vmStatses, nil) 1641 1642 outp.puts 1643 printVMs(outp) 1644 outp.puts "Geomean of preferred means:" 1645 outp.print " " 1646 outp.print rpad("<scaled-result>", $benchpad) 1647 outp.print " " 1648 $vms.size.times { 1649 | index | 1650 if index != 0 1651 outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm 1652 end 1653 outp.print statsToStr(overallResults[index]) 1654 } 1655 if overallResults.size>=2 1656 outp.print(" "+overallResults[-1].compareTo(overallResults[0]).longForm) 1657 end 1658 outp.puts 1659 end 1660 outp.puts 1661 1662 if outp != $stdout 1663 outp.close 1664 end 1665 1666 if outp != $stdout and not $brief 1667 puts 1668 File.open(reportName) { 1669 | inp | 1670 puts inp.read 1671 } 1672 end 1673 1674 if $brief 1675 puts(overallResults.collect{|stats| stats.mean}.join("\t")) 1676 puts(overallResults.collect{|stats| stats.confInt}.join("\t")) 1677 end 1678 1679 1680 end 1681 1682 begin 1683 $sawBenchOptions = false 1684 1685 def resetBenchOptionsIfNecessary 1686 unless $sawBenchOptions 1687 $includeSunSpider = false 1688 $includeV8 = false 1689 $includeKraken = false 1690 $sawBenchOptions = true 1691 end 1692 end 1693 1694 GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT], 1695 ['--inner', GetoptLong::REQUIRED_ARGUMENT], 1696 ['--outer', GetoptLong::REQUIRED_ARGUMENT], 1697 ['--warmup', GetoptLong::REQUIRED_ARGUMENT], 1698 ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT], 1699 ['--sunspider-only', GetoptLong::NO_ARGUMENT], 1700 ['--v8-only', GetoptLong::NO_ARGUMENT], 1701 ['--kraken-only', GetoptLong::NO_ARGUMENT], 1702 ['--exclude-sunspider', GetoptLong::NO_ARGUMENT], 1703 ['--exclude-v8', GetoptLong::NO_ARGUMENT], 1704 ['--exclude-kraken', GetoptLong::NO_ARGUMENT], 1705 ['--sunspider', GetoptLong::NO_ARGUMENT], 1706 ['--v8', GetoptLong::NO_ARGUMENT], 1707 ['--kraken', GetoptLong::NO_ARGUMENT], 1708 ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT], 1709 ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT], 1710 ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT], 1711 ['--force-vm-copy', GetoptLong::NO_ARGUMENT], 1712 ['--dont-copy-vms', GetoptLong::NO_ARGUMENT], 1713 ['--verbose', '-v', GetoptLong::NO_ARGUMENT], 1714 ['--brief', GetoptLong::NO_ARGUMENT], 1715 ['--silent', GetoptLong::NO_ARGUMENT], 1716 ['--remote', GetoptLong::REQUIRED_ARGUMENT], 1717 ['--local', GetoptLong::NO_ARGUMENT], 1718 ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT], 1719 ['--slave', GetoptLong::NO_ARGUMENT], 1720 ['--prepare-only', GetoptLong::NO_ARGUMENT], 1721 ['--analyze', GetoptLong::REQUIRED_ARGUMENT], 1722 ['--vms', GetoptLong::REQUIRED_ARGUMENT], 1723 ['--help', '-h', GetoptLong::NO_ARGUMENT]).each { 1724 | opt, arg | 1725 case opt 1726 when '--rerun' 1727 $rerun = intArg(opt,arg,1,nil) 1728 when '--inner' 1729 $inner = intArg(opt,arg,1,nil) 1730 when '--outer' 1731 $outer = intArg(opt,arg,1,nil) 1732 when '--warmup' 1733 $warmup = intArg(opt,arg,0,nil) 1734 when '--timing-mode' 1735 if arg.upcase == "PRECISETIME" 1736 $timeMode = :preciseTime 1737 elsif arg.upcase == "DATE" 1738 $timeMode = :date 1739 elsif arg.upcase == "AUTO" 1740 $timeMode = :auto 1741 else 1742 quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.", 1743 "Invalid argument for command-line option") 1744 end 1745 when '--force-vm-kind' 1746 if arg.upcase == "JSC" 1747 $forceVMKind = :jsc 1748 elsif arg.upcase == "DUMPRENDERTREE" 1749 $forceVMKind = :dumpRenderTree 1750 elsif arg.upcase == "AUTO" 1751 $forceVMKind = nil 1752 else 1753 quickFail("Expected either 'jsc' or 'DumpRenderTree' for --force-vm-kind, but got '#{arg}'.", 1754 "Invalid argument for command-line option") 1755 end 1756 when '--force-vm-copy' 1757 $needToCopyVMs = true 1758 when '--dont-copy-vms' 1759 $dontCopyVMs = true 1760 when '--sunspider-only' 1761 $includeV8 = false 1762 $includeKraken = false 1763 when '--v8-only' 1764 $includeSunSpider = false 1765 $includeKraken = false 1766 when '--kraken-only' 1767 $includeSunSpider = false 1768 $includeV8 = false 1769 when '--exclude-sunspider' 1770 $includeSunSpider = false 1771 when '--exclude-v8' 1772 $includeV8 = false 1773 when '--exclude-kraken' 1774 $includeKraken = false 1775 when '--sunspider' 1776 resetBenchOptionsIfNecessary 1777 $includeSunSpider = true 1778 when '--v8' 1779 resetBenchOptionsIfNecessary 1780 $includeV8 = true 1781 when '--kraken' 1782 resetBenchOptionsIfNecessary 1783 $includeKraken = true 1784 when '--benchmarks' 1785 $benchmarkPattern = Regexp.new(arg) 1786 when '--measure-gc' 1787 if arg == '' 1788 $measureGC = true 1789 else 1790 $measureGC = arg 1791 end 1792 when '--verbose' 1793 $verbosity += 1 1794 when '--brief' 1795 $brief = true 1796 when '--silent' 1797 $silent = true 1798 when '--remote' 1799 $remoteHosts += arg.split(',') 1800 $needToCopyVMs = true 1801 when '--ssh-options' 1802 $sshOptions << arg 1803 when '--local' 1804 $alsoLocal = true 1805 when '--prepare-only' 1806 $run = false 1807 when '--analyze' 1808 $prepare = false 1809 $run = false 1810 $analyze << arg 1811 when '--help' 1812 usage 1813 else 1814 raise "bad option: #{opt}" 1815 end 1816 } 1817 1818 # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option. 1819 if $dontCopyVMs 1820 $needToCopyVMs = false 1821 end 1822 1823 SUNSPIDER = BenchmarkSuite.new("SunSpider", SUNSPIDER_PATH, :arithmeticMean) 1824 ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees", 1825 "access-fannkuch", "access-nbody", "access-nsieve", 1826 "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and", 1827 "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes", 1828 "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb", 1829 "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna", 1830 "string-base64", "string-fasta", "string-tagcloud", 1831 "string-unpack-code", "string-validate-input"].each { 1832 | name | 1833 SUNSPIDER.add SunSpiderBenchmark.new(name) 1834 } 1835 1836 V8 = BenchmarkSuite.new("V8", V8_PATH, :geometricMean) 1837 ["crypto", "deltablue", "earley-boyer", "raytrace", 1838 "regexp", "richards", "splay"].each { 1839 | name | 1840 V8.add V8Benchmark.new(name) 1841 } 1842 1843 KRAKEN = BenchmarkSuite.new("Kraken", KRAKEN_PATH, :arithmeticMean) 1844 ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft", 1845 "audio-oscillator", "imaging-darkroom", "imaging-desaturate", 1846 "imaging-gaussian-blur", "json-parse-financial", 1847 "json-stringify-tinderbox", "stanford-crypto-aes", 1848 "stanford-crypto-ccm", "stanford-crypto-pbkdf2", 1849 "stanford-crypto-sha256-iterative"].each { 1850 | name | 1851 KRAKEN.add KrakenBenchmark.new(name) 1852 } 1853 1854 ARGV.each { 1855 | vm | 1856 if vm =~ /([a-zA-Z0-9_ ]+):/ 1857 name = $1 1858 nameKind = :given 1859 vm = $~.post_match 1860 else 1861 name = "Conf\##{$vms.length+1}" 1862 nameKind = :auto 1863 end 1864 $stderr.puts "#{name}: #{vm}" if $verbosity >= 1 1865 $vms << VM.new(Pathname.new(vm).realpath, name, nameKind, nil) 1866 } 1867 1868 if $vms.empty? 1869 quickFail("Please specify at least on configuraiton on the command line.", 1870 "Insufficient arguments") 1871 end 1872 1873 $vms.each { 1874 | vm | 1875 if vm.vmType != :jsc and $timeMode != :date 1876 $timeMode = :date 1877 $stderr.puts "Warning: using Date.now() instead of preciseTime() because #{vm} doesn't support the latter." 1878 end 1879 } 1880 1881 if FileTest.exist? BENCH_DATA_PATH 1882 cmd = "rm -rf #{BENCH_DATA_PATH}" 1883 $stderr.puts ">> #{cmd}" if $verbosity >= 2 1884 raise unless system cmd 1885 end 1886 1887 Dir.mkdir BENCH_DATA_PATH 1888 1889 if $needToCopyVMs 1890 canCopyIntoBenchPath = true 1891 $vms.each { 1892 | vm | 1893 canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath 1894 } 1895 1896 if canCopyIntoBenchPath 1897 $vms.each { 1898 | vm | 1899 $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..." 1900 vm.copyIntoBenchPath 1901 } 1902 $stderr.puts "All VMs are in place." 1903 else 1904 $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it." 1905 end 1906 end 1907 1908 if $measureGC and $measureGC != true 1909 found = false 1910 $vms.each { 1911 | vm | 1912 if vm.name == $measureGC 1913 found = true 1914 end 1915 } 1916 unless found 1917 $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}" 1918 end 1919 end 1920 1921 if $outer*$inner == 1 1922 $stderr.puts "Warning: will only collect one sample per benchmark/VM. Confidence interval calculation will fail." 1923 end 1924 1925 $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1 1926 1927 $suites = [] 1928 1929 if $includeSunSpider and not SUNSPIDER.empty? 1930 $suites << SUNSPIDER 1931 end 1932 1933 if $includeV8 and not V8.empty? 1934 $suites << V8 1935 end 1936 1937 if $includeKraken and not KRAKEN.empty? 1938 $suites << KRAKEN 1939 end 1940 1941 $benchmarks = [] 1942 $suites.each { 1943 | suite | 1944 $benchmarks += suite.benchmarks 1945 } 1946 1947 $runPlans = [] 1948 $vms.each { 1949 | vm | 1950 $benchmarks.each { 1951 | benchmark | 1952 $outer.times { 1953 | iteration | 1954 $runPlans << BenchRunPlan.new(benchmark, vm, iteration) 1955 } 1956 } 1957 } 1958 1959 $runPlans.shuffle! 1960 1961 $suitepad = $suites.collect { 1962 | suite | 1963 suite.to_s.size 1964 }.max + 1 1965 1966 $benchpad = ($benchmarks + 1967 ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect { 1968 | benchmark | 1969 if benchmark.respond_to? :name 1970 benchmark.name.size 1971 else 1972 benchmark.size 1973 end 1974 }.max + 1 1975 1976 $vmpad = $vms.collect { 1977 | vm | 1978 vm.to_s.size 1979 }.max + 1 1980 1981 if $prepare 1982 File.open("#{BENCH_DATA_PATH}/runscript", "w") { 1983 | file | 1984 file.puts "echo \"HOSTNAME:\\c\"" 1985 file.puts "hostname" 1986 file.puts "echo" 1987 file.puts "echo \"HARDWARE:\\c\"" 1988 file.puts "/usr/sbin/sysctl hw.model" 1989 file.puts "echo" 1990 file.puts "set -e" 1991 $script = file 1992 $runPlans.each_with_index { 1993 | plan, idx | 1994 if $verbosity == 0 and not $silent 1995 text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s 1996 text2 = plan.benchmark.to_s+"/"+plan.vm.to_s 1997 file.puts("echo "+("\r#{text1} #{rpad(text2,$suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2") 1998 file.puts("echo "+("\r#{text1} #{text2}".inspect)[0..-2]+"\\c\" 1>&2") 1999 end 2000 plan.emitRunCode 2001 } 2002 if $verbosity == 0 and not $silent 2003 file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2") 2004 file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size}".inspect)+" 1>&2") 2005 end 2006 } 2007 end 2008 2009 if $run 2010 unless $remoteHosts.empty? 2011 $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0 2012 Dir.chdir(TEMP_PATH) { 2013 cmd = "tar -czf payload.tar.gz benchdata" 2014 $stderr.puts ">> #{cmd}" if $verbosity>=2 2015 raise unless system(cmd) 2016 } 2017 2018 def grokHost(host) 2019 if host =~ /:([0-9]+)$/ 2020 "-p " + $1 + " " + $~.pre_match.inspect 2021 else 2022 host.inspect 2023 end 2024 end 2025 2026 def sshRead(host, command) 2027 cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}" 2028 $stderr.puts ">> #{cmd}" if $verbosity>=2 2029 result = "" 2030 IO.popen(cmd, "r") { 2031 | inp | 2032 inp.each_line { 2033 | line | 2034 $stderr.puts "#{host}: #{line}" if $verbosity>=2 2035 result += line 2036 } 2037 } 2038 raise "#{$?}" unless $?.success? 2039 result 2040 end 2041 2042 def sshWrite(host, command, data) 2043 cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}" 2044 $stderr.puts ">> #{cmd}" if $verbosity>=2 2045 IO.popen(cmd, "w") { 2046 | outp | 2047 outp.write(data) 2048 } 2049 raise "#{$?}" unless $?.success? 2050 end 2051 2052 $remoteHosts.each { 2053 | host | 2054 $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0 2055 2056 remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"] 2057 raise unless remoteTempPath 2058 2059 sshWrite(host, "cd #{remoteTempPath.inspect} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz")) 2060 2061 $stderr.puts "Running on #{host}..." if $verbosity==0 2062 2063 parseAndDisplayResults(sshRead(host, "cd #{(remoteTempPath+'/benchdata').inspect} && sh runscript")) 2064 } 2065 end 2066 2067 if not $remoteHosts.empty? and $alsoLocal 2068 $stderr.puts "Running locally..." 2069 end 2070 2071 if $remoteHosts.empty? or $alsoLocal 2072 parseAndDisplayResults(runAndGetResults) 2073 end 2074 end 2075 2076 $analyze.each_with_index { 2077 | filename, index | 2078 if index >= 1 2079 puts 2080 end 2081 parseAndDisplayResults(IO::read(filename)) 2082 } 2083 2084 if $prepare and not $run and $analyze.empty? 2085 puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+ 2086 "the benchmarks and get the results by doing:", 78) 2087 puts 2088 puts "cd #{BENCH_DATA_PATH}" 2089 puts "sh runscript > results.txt" 2090 puts 2091 puts wrap("Then you can analyze the results by running bencher with the same arguments "+ 2092 "as now, but replacing --prepare-only with --analyze results.txt.", 78) 2093 end 2094 rescue => e 2095 fail(e) 2096 end 2097 2098 2099
