1 /* 2 * Copyright (C) 2013 Google Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 * in compliance with the License. You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software distributed under the License 10 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 * or implied. See the License for the specific language governing permissions and limitations under 12 * the License. 13 */ 14 15 package com.google.caliper.runner; 16 17 import static org.junit.Assert.assertTrue; 18 import static org.junit.Assert.fail; 19 20 import com.google.caliper.BeforeExperiment; 21 import com.google.caliper.Benchmark; 22 import com.google.common.collect.Lists; 23 24 import org.junit.Rule; 25 import org.junit.Test; 26 import org.junit.runner.RunWith; 27 import org.junit.runners.JUnit4; 28 29 import java.util.List; 30 31 /** 32 * Integration tests for misbehaving benchmarks. 33 */ 34 @RunWith(JUnit4.class) 35 public class BadUserCodeTest { 36 @Rule public CaliperTestWatcher runner = new CaliperTestWatcher(); 37 38 @Test 39 40 public void testExceptionInInit() throws Exception { 41 try { 42 runner.forBenchmark(ExceptionInInitBenchmark.class).run(); 43 fail(); 44 } catch (UserCodeException expected) {} 45 } 46 47 private static void throwSomeUserException() { 48 throw new RuntimeException(); 49 } 50 51 static class ExceptionInInitBenchmark { 52 static { 53 throwSomeUserException(); 54 } 55 56 @Benchmark void timeSomething(int reps) { 57 fail("" + reps); 58 } 59 } 60 61 @Test 62 63 public void testExceptionInConstructor() throws Exception { 64 try { 65 runner.forBenchmark(ExceptionInConstructorBenchmark.class).run(); 66 fail(); 67 } catch (UserCodeException expected) {} 68 } 69 70 static class ExceptionInConstructorBenchmark { 71 ExceptionInConstructorBenchmark() { 72 throw new RuntimeException(); 73 } 74 75 @Benchmark void timeSomething(int reps) { 76 fail("" + reps); 77 } 78 } 79 80 @Test 81 82 public void testExceptionInMethod() throws Exception { 83 try { 84 runner.forBenchmark(ExceptionInMethodBenchmark.class).run(); 85 fail(); 86 } catch (UserCodeException expected) {} 87 } 88 89 static class ExceptionInMethodBenchmark { 90 @Benchmark void timeSomething(@SuppressWarnings("unused") int reps) { 91 throw new RuntimeException(); 92 } 93 } 94 95 @Test 96 97 public void testExceptionInMethod_notInDryRun() throws Exception { 98 try { 99 runner.forBenchmark(ExceptionLateInMethodBenchmark.class).run(); 100 fail(); 101 } catch (ProxyWorkerException expected) { 102 assertTrue(expected.getMessage().contains(ExceptionLateInMethodBenchmark.class.getName())); 103 } 104 } 105 106 static class ExceptionLateInMethodBenchmark { 107 @Benchmark void timeSomething(int reps) { 108 if (reps > 1) { 109 throw new RuntimeException(); 110 } 111 } 112 } 113 114 @Test 115 116 public void testExceptionInSetUp() throws Exception { 117 try { 118 runner.forBenchmark(ExceptionInSetUpBenchmark.class).run(); 119 fail(); 120 } catch (UserCodeException expected) {} 121 } 122 123 static class ExceptionInSetUpBenchmark { 124 @BeforeExperiment void setUp() { 125 throw new RuntimeException(); 126 } 127 128 @Benchmark void timeSomething(int reps) { 129 fail("" + reps); 130 } 131 } 132 133 @Test 134 135 public void testNonDeterministicAllocation_noTrackAllocations() throws Exception { 136 try { 137 runner.forBenchmark(NonDeterministicAllocationBenchmark.class) 138 .instrument("allocation") 139 .options("-Cinstrument.allocation.options.trackAllocations=" + false) 140 .run(); 141 fail(); 142 } catch (ProxyWorkerException expected) { 143 String message = "Your benchmark appears to have non-deterministic allocation behavior"; 144 assertTrue("Expected " + expected.getMessage() + " to contain " + message, 145 expected.getMessage().contains(message)); 146 } 147 } 148 149 @Test 150 151 public void testNonDeterministicAllocation_trackAllocations() throws Exception { 152 try { 153 runner.forBenchmark(NonDeterministicAllocationBenchmark.class) 154 .instrument("allocation") 155 .options("-Cinstrument.allocation.options.trackAllocations=" + true) 156 .run(); 157 fail(); 158 } catch (ProxyWorkerException expected) { 159 String message = "Your benchmark appears to have non-deterministic allocation behavior"; 160 assertTrue("Expected " + expected.getMessage() + " to contain " + message, 161 expected.getMessage().contains(message)); 162 } 163 } 164 165 /** The number of allocations is non deterministic because it depends on static state. */ 166 static class NonDeterministicAllocationBenchmark { 167 static int timeCount = 0; 168 // We dump items into this list so the jit cannot remove the allocations 169 static List<Object> list = Lists.newArrayList(); 170 @Benchmark int timeSomethingFBZ(@SuppressWarnings("unused") int reps) { 171 timeCount++; 172 if (timeCount % 2 == 0) { 173 list.add(new Object()); 174 return list.hashCode(); 175 } 176 return this.hashCode(); 177 } 178 } 179 180 @Test 181 182 public void testComplexNonDeterministicAllocation_noTrackAllocations() throws Exception { 183 // Without trackAllocations enabled this kind of non-determinism cannot be detected. 184 runner.forBenchmark(ComplexNonDeterministicAllocationBenchmark.class) 185 .instrument("allocation") 186 .options("-Cinstrument.allocation.options.trackAllocations=" + false) 187 .run(); 188 } 189 190 @Test 191 192 public void testComplexNonDeterministicAllocation_trackAllocations() throws Exception { 193 try { 194 runner.forBenchmark(ComplexNonDeterministicAllocationBenchmark.class) 195 .instrument("allocation") 196 .options("-Cinstrument.allocation.options.trackAllocations=" + true) 197 .run(); 198 } catch (ProxyWorkerException expected) { 199 String message = "Your benchmark appears to have non-deterministic allocation behavior"; 200 assertTrue("Expected " + expected.getMessage() + " to contain " + message, 201 expected.getMessage().contains(message)); 202 } 203 } 204 205 /** Benchmark allocates the same number of things each time but in a different way. */ 206 static class ComplexNonDeterministicAllocationBenchmark { 207 static int timeCount = 0; 208 @Benchmark int timeSomethingFBZ(@SuppressWarnings("unused") int reps) { 209 // We dump items into this list so the jit cannot remove the allocations 210 List<Object> list = Lists.newArrayList(); 211 timeCount++; 212 if (timeCount % 2 == 0) { 213 list.add(new Object()); 214 } else { 215 list.add(new Object()); 216 } 217 return list.hashCode(); 218 } 219 } 220 } 221
