1 /** 2 * Copyright (C) 2009 Google Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.google.caliper.runner; 18 19 import com.google.caliper.model.BenchmarkSpec; 20 import com.google.caliper.model.InstrumentSpec; 21 import com.google.caliper.model.Measurement; 22 import com.google.caliper.model.Scenario; 23 import com.google.caliper.model.Trial; 24 import com.google.caliper.model.VmSpec; 25 import com.google.caliper.util.Stdout; 26 import com.google.common.base.Function; 27 import com.google.common.base.Stopwatch; 28 import com.google.common.collect.FluentIterable; 29 import com.google.common.collect.ImmutableListMultimap; 30 import com.google.common.collect.ImmutableSet; 31 import com.google.common.collect.Iterables; 32 import com.google.common.collect.Multimaps; 33 import com.google.common.collect.Ordering; 34 import com.google.common.collect.Sets; 35 36 import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; 37 import org.apache.commons.math.stat.descriptive.rank.Percentile; 38 39 import java.io.Closeable; 40 import java.io.PrintWriter; 41 import java.util.Collection; 42 import java.util.Map.Entry; 43 import java.util.Set; 44 45 /** 46 * Prints a brief summary of the results collected. It does not contain the measurements themselves 47 * as that is the responsibility of the webapp. 48 */ 49 final class ConsoleOutput implements Closeable { 50 private final PrintWriter stdout; 51 52 private final Set<InstrumentSpec> instrumentSpecs = Sets.newHashSet(); 53 private final Set<VmSpec> vmSpecs = Sets.newHashSet(); 54 private final Set<BenchmarkSpec> benchmarkSpecs = Sets.newHashSet(); 55 private int numMeasurements = 0; 56 private int trialsCompleted = 0; 57 private final int numberOfTrials; 58 private final Stopwatch stopwatch; 59 60 61 ConsoleOutput(@Stdout PrintWriter stdout, int numberOfTrials, Stopwatch stopwatch) { 62 this.stdout = stdout; 63 this.numberOfTrials = numberOfTrials; 64 this.stopwatch = stopwatch; 65 } 66 67 /** 68 * Prints a short message when we observe a trial failure. 69 */ 70 void processFailedTrial(TrialFailureException e) { 71 trialsCompleted++; 72 // TODO(lukes): it would be nice to print which trial failed. Consider adding Experiment data 73 // to the TrialFailureException. 74 stdout.println( 75 "ERROR: Trial failed to complete (its results will not be included in the run):\n" 76 + " " + e.getMessage()); 77 stdout.flush(); 78 } 79 80 /** 81 * Prints a summary of a successful trial result. 82 */ 83 void processTrial(TrialResult result) { 84 trialsCompleted++; 85 stdout.printf("Trial Report (%d of %d):%n Experiment %s%n", 86 trialsCompleted, numberOfTrials, result.getExperiment()); 87 if (!result.getTrialMessages().isEmpty()) { 88 stdout.println(" Messages:"); 89 for (String message : result.getTrialMessages()) { 90 stdout.print(" "); 91 stdout.println(message); 92 } 93 } 94 Trial trial = result.getTrial(); 95 ImmutableListMultimap<String, Measurement> measurementsIndex = 96 new ImmutableListMultimap.Builder<String, Measurement>() 97 .orderKeysBy(Ordering.natural()) 98 .putAll(Multimaps.index(trial.measurements(), new Function<Measurement, String>() { 99 @Override public String apply(Measurement input) { 100 return input.description(); 101 } 102 })) 103 .build(); 104 stdout.println(" Results:"); 105 for (Entry<String, Collection<Measurement>> entry : measurementsIndex.asMap().entrySet()) { 106 Collection<Measurement> measurements = entry.getValue(); 107 ImmutableSet<String> units = FluentIterable.from(measurements) 108 .transform(new Function<Measurement, String>() { 109 @Override public String apply(Measurement input) { 110 return input.value().unit(); 111 } 112 }).toSet(); 113 double[] weightedValues = new double[measurements.size()]; 114 int i = 0; 115 for (Measurement measurement : measurements) { 116 weightedValues[i] = measurement.value().magnitude() / measurement.weight(); 117 i++; 118 } 119 Percentile percentile = new Percentile(); 120 percentile.setData(weightedValues); 121 DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics(weightedValues); 122 String unit = Iterables.getOnlyElement(units); 123 stdout.printf( 124 " %s%s: min=%.2f, 1st qu.=%.2f, median=%.2f, mean=%.2f, 3rd qu.=%.2f, max=%.2f%n", 125 entry.getKey(), unit.isEmpty() ? "" : "(" + unit + ")", 126 descriptiveStatistics.getMin(), percentile.evaluate(25), 127 percentile.evaluate(50), descriptiveStatistics.getMean(), 128 percentile.evaluate(75), descriptiveStatistics.getMax()); 129 } 130 131 instrumentSpecs.add(trial.instrumentSpec()); 132 Scenario scenario = trial.scenario(); 133 vmSpecs.add(scenario.vmSpec()); 134 benchmarkSpecs.add(scenario.benchmarkSpec()); 135 numMeasurements += trial.measurements().size(); 136 } 137 138 @Override public void close() { 139 if (trialsCompleted == numberOfTrials) { // if we finished all the trials 140 stdout.printf("Collected %d measurements from:%n", numMeasurements); 141 stdout.printf(" %d instrument(s)%n", instrumentSpecs.size()); 142 stdout.printf(" %d virtual machine(s)%n", vmSpecs.size()); 143 stdout.printf(" %d benchmark(s)%n", benchmarkSpecs.size()); 144 stdout.println(); 145 stdout.format("Execution complete: %s.%n", stopwatch.stop()); 146 stdout.flush(); 147 } 148 } 149 } 150
