Home | History | Annotate | Download | only in runner 
      1 /*
      2  * Copyright (C) 2009 Google Inc.
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  * http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.google.caliper.runner;
     18 
     19 import static org.junit.Assert.assertEquals;
     20 import static org.junit.Assert.fail;
     21 
     22 import com.google.caliper.Benchmark;
     23 import com.google.caliper.Param;
     24 import com.google.caliper.config.InvalidConfigurationException;
     25 import com.google.caliper.util.InvalidCommandException;
     26 
     27 import junit.framework.AssertionFailedError;
     28 
     29 import org.junit.Test;
     30 import org.junit.runner.RunWith;
     31 import org.junit.runners.JUnit4;
     32 
     33 import java.io.PrintWriter;
     34 import java.io.StringWriter;
     35 
     36 /**
     37  * Unit test covering common user mistakes in benchmark classes.
     38  */
     39 @RunWith(JUnit4.class)
     40 
     41 public class MalformedBenchmarksTest {
     42   // Put the expected messages together here, which may promote some kind of
     43   // consistency in their wording. :)
     44 
     45   private static final String ABSTRACT =
     46       "Class '%s' is abstract";
     47   private static final String NO_CONSTRUCTOR =
     48       "Benchmark class %s does not have a publicly visible default constructor";
     49   private static final String NO_METHODS =
     50       "There were no experiments to be performed for the class %s using the instruments " +
     51       "[allocation, runtime]";
     52   private static final String STATIC_BENCHMARK =
     53       "Benchmark methods must not be static: timeIt";
     54   private static final String WRONG_ARGUMENTS =
     55       "Benchmark methods must have no arguments or accept a single int or long parameter: timeIt";
     56   private static final String STATIC_PARAM =
     57       "Parameter field 'oops' must not be static";
     58   private static final String RESERVED_PARAM =
     59       "Class '%s' uses reserved parameter name 'vm'";
     60   private static final String NO_CONVERSION = "Type 'Object' of parameter field 'oops' "
     61       + "has no recognized String-converting method; see <TODO> for details";
     62   private static final String CONVERT_FAILED = // granted this one's a little weird (and brittle)
     63       "Cannot convert value 'oops' to type 'int': For input string: \"oops\"";
     64 
     65   @Test public void abstractBenchmark() throws Exception {
     66     expectException(ABSTRACT, AbstractBenchmark.class);
     67   }
     68   abstract static class AbstractBenchmark {}
     69 
     70   @Test public void noSuitableConstructor() throws Exception {
     71     expectException(String.format(NO_CONSTRUCTOR, BadConstructorBenchmark.class.getName()),
     72         BadConstructorBenchmark.class);
     73   }
     74 
     75   @SuppressWarnings("unused")
     76   static class BadConstructorBenchmark {
     77     BadConstructorBenchmark(String damnParam) {}
     78     @Benchmark void timeIt(int reps) {}
     79   }
     80 
     81   @Test public void noBenchmarkMethods() throws Exception {
     82     expectException(NO_METHODS, NoMethodsBenchmark.class);
     83   }
     84 
     85   @SuppressWarnings("unused")
     86   static class NoMethodsBenchmark {
     87     void timeIt(int reps) {} // not annotated
     88   }
     89 
     90   @Test public void staticBenchmarkMethod() throws Exception {
     91     expectException(STATIC_BENCHMARK, StaticBenchmarkMethodBenchmark.class);
     92   }
     93 
     94   @SuppressWarnings("unused")
     95   static class StaticBenchmarkMethodBenchmark {
     96     @Benchmark public static void timeIt(int reps) {}
     97   }
     98 
     99   @Test public void wrongSignature() throws Exception {
    100     expectException(WRONG_ARGUMENTS, BoxedParamBenchmark.class);
    101     expectException(WRONG_ARGUMENTS, ExtraParamBenchmark.class);
    102   }
    103 
    104   @SuppressWarnings("unused")
    105   static class BoxedParamBenchmark {
    106     @Benchmark void timeIt(Integer reps) {}
    107   }
    108 
    109   @SuppressWarnings("unused")
    110   static class ExtraParamBenchmark {
    111     @Benchmark void timeIt(int reps, int what) {}
    112   }
    113 
    114   @Test public void hasBenchmarkOverloads() throws Exception {
    115     // N.B. baz is fine since although it has an overload, its overload is not a benchmark method.
    116     expectException(
    117         "Overloads are disallowed for benchmark methods, found overloads of [bar, foo] in "
    118         + "benchmark OverloadsAnnotatedBenchmark",
    119         OverloadsAnnotatedBenchmark.class);
    120   }
    121 
    122   @SuppressWarnings("unused")
    123   static class OverloadsAnnotatedBenchmark {
    124     @Benchmark public void foo(long reps) {}
    125     @Benchmark public void foo(int reps) {}
    126     @Benchmark public void bar(long reps) {}
    127     @Benchmark public void bar(int reps) {}
    128     @Benchmark public void baz(int reps) {}
    129     public void baz(long reps, boolean thing) {}
    130     public void baz(long reps) {}
    131   }
    132 
    133   @Test public void staticParam() throws Exception {
    134     expectException(STATIC_PARAM, StaticParamBenchmark.class);
    135   }
    136   static class StaticParamBenchmark {
    137     @Param static String oops;
    138   }
    139 
    140   @Test public void reservedParameterName() throws Exception {
    141     expectException(RESERVED_PARAM, ReservedParamBenchmark.class);
    142   }
    143   static class ReservedParamBenchmark {
    144     @Param String vm;
    145   }
    146 
    147   @Test public void unparsableParamType() throws Exception {
    148     expectException(NO_CONVERSION, UnparsableParamTypeBenchmark.class);
    149   }
    150   static class UnparsableParamTypeBenchmark {
    151     @Param Object oops;
    152   }
    153 
    154   @Test public void unparsableParamDefault() throws Exception {
    155     expectException(CONVERT_FAILED, UnparsableParamDefaultBenchmark.class);
    156   }
    157   static class UnparsableParamDefaultBenchmark {
    158     @Param({"1", "2", "oops"}) int number;
    159   }
    160 
    161   // end of tests
    162 
    163   private void expectException(String expectedMessageFmt, Class<?> benchmarkClass)
    164       throws InvalidCommandException, InvalidConfigurationException {
    165     try {
    166       CaliperMain.exitlessMain(
    167           new String[] {"--instrument=allocation,runtime", "--dry-run", benchmarkClass.getName()},
    168           new PrintWriter(new StringWriter()), new PrintWriter(new StringWriter()));
    169       fail("no exception thrown");
    170     } catch (InvalidBenchmarkException e) {
    171       try {
    172         String expectedMessageText =
    173             String.format(expectedMessageFmt, benchmarkClass.getSimpleName());
    174         assertEquals(expectedMessageText, e.getMessage());
    175 
    176         // don't swallow our real stack trace
    177       } catch (AssertionFailedError afe) {
    178         afe.initCause(e);
    179         throw afe;
    180       }
    181     }
    182   }
    183 }
    184