1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.metricregression; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertTrue; 20 import static org.mockito.ArgumentMatchers.any; 21 import static org.mockito.ArgumentMatchers.eq; 22 import static org.mockito.Mockito.doNothing; 23 import static org.mockito.Mockito.spy; 24 import static org.mockito.Mockito.times; 25 import static org.mockito.Mockito.verify; 26 27 import com.android.tradefed.result.TestDescription; 28 import com.android.tradefed.testtype.metricregression.DetectRegression.TableRow; 29 import com.android.tradefed.util.MultiMap; 30 31 import com.google.common.primitives.Doubles; 32 33 import org.junit.Test; 34 import org.junit.runner.RunWith; 35 import org.junit.runners.JUnit4; 36 import org.mockito.ArgumentCaptor; 37 38 import java.util.Arrays; 39 import java.util.HashMap; 40 import java.util.List; 41 import java.util.Map; 42 import java.util.stream.Collectors; 43 44 /** Unit tests for {@link DetectRegression}. */ 45 @RunWith(JUnit4.class) 46 public class DetectRegressionTest { 47 48 private static final double EPS = 0.0001; 49 50 @Test 51 public void testCalcMean() { 52 Map<Double, double[]> data = new HashMap<>(); 53 data.put(2.5, new double[] {1, 2, 3, 4}); 54 data.put( 55 4.5, 56 new double[] { 57 -11, 22, 5, 4, 2.5, 58 }); 59 data.forEach( 60 (k, v) -> { 61 assertTrue(equal(k, DetectRegression.calcMean(Doubles.asList(v)))); 62 }); 63 } 64 65 @Test 66 public void testCalcStdDev() { 67 Map<Double, double[]> data = new HashMap<>(); 68 data.put(36.331000536732, new double[] {12.3, 56.7, 45.6, 124, 56}); 69 data.put(119.99906922093, new double[] {123.4, 22.5, 5.67, 4.56, 2.5, 333}); 70 data.forEach( 71 (k, v) -> { 72 assertTrue(equal(k, DetectRegression.calcStdDev(Doubles.asList(v)))); 73 }); 74 } 75 76 @Test 77 public void testDetectRegression() { 78 List<List<Double>> befores = 79 Arrays.stream( 80 new double[][] { 81 {3, 3, 3, 3, 3}, 82 {3, 5, 3, 5, 4}, 83 {3, 4, 3, 4, 3}, 84 {1, 2, 3, 2, 1}, 85 {-1, -2, -3, 0, 1, 2, 3}, 86 {5, 6, 5, 6, 6, 5, 7}, 87 }) 88 .map(Doubles::asList) 89 .collect(Collectors.toList()); 90 List<List<Double>> afters = 91 Arrays.stream( 92 new double[][] { 93 {3, 3, 3, 3, 3}, 94 {2, 3, 4, 5, 6}, 95 {2, 5, 2, 5, 2}, 96 {10, 11, 12, 13, 14}, 97 {-10, -20, -30, 0, 10, 20, 30}, 98 {5, 6, 5, 6, 6, 5, 700}, 99 }) 100 .map(Doubles::asList) 101 .collect(Collectors.toList()); 102 boolean[] result = {false, false, true, true, true, false}; 103 104 for (int i = 0; i < result.length; i++) { 105 assertEquals( 106 result[i], DetectRegression.computeRegression(befores.get(i), afters.get(i))); 107 } 108 } 109 110 @SuppressWarnings("unchecked") 111 @Test 112 public void testRunRegressionDetection() { 113 DetectRegression detector = spy(DetectRegression.class); 114 doNothing().when(detector).logResult(any(), any(), any(), any()); 115 TestDescription id1 = new TestDescription("class", "test1"); 116 TestDescription id2 = new TestDescription("class", "test2"); 117 Metrics before = new Metrics(false); 118 Arrays.asList("3.0", "3.0", "3.0", "3.0", "3.0") 119 .forEach(e -> before.addRunMetric("metric-1", e)); 120 Arrays.asList("3.1", "3.3", "3.1", "3.2", "3.3") 121 .forEach(e -> before.addRunMetric("metric-2", e)); 122 Arrays.asList("5.1", "5.2", "5.1", "5.2", "5.1") 123 .forEach(e -> before.addRunMetric("metric-3", e)); 124 Arrays.asList("3.0", "3.0", "3.0", "3.0", "3.0") 125 .forEach(e -> before.addTestMetric(id1, "metric-4", e)); 126 Arrays.asList("3.1", "3.3", "3.1", "3.2", "3.3") 127 .forEach(e -> before.addTestMetric(id2, "metric-5", e)); 128 Arrays.asList("5.1", "5.2", "5.1", "5.2", "5.1") 129 .forEach(e -> before.addTestMetric(id2, "metric-6", e)); 130 131 Metrics after = new Metrics(false); 132 Arrays.asList("3.0", "3.0", "3.0", "3.0", "3.0") 133 .forEach(e -> after.addRunMetric("metric-1", e)); 134 Arrays.asList("3.2", "3.2", "3.2", "3.2", "3.2") 135 .forEach(e -> after.addRunMetric("metric-2", e)); 136 Arrays.asList("8.1", "8.2", "8.1", "8.2", "8.1") 137 .forEach(e -> after.addRunMetric("metric-3", e)); 138 Arrays.asList("3.0", "3.0", "3.0", "3.0", "3.0") 139 .forEach(e -> after.addTestMetric(id1, "metric-4", e)); 140 Arrays.asList("3.2", "3.2", "3.2", "3.2", "3.2") 141 .forEach(e -> after.addTestMetric(id2, "metric-5", e)); 142 Arrays.asList("8.1", "8.2", "8.1", "8.2", "8.1") 143 .forEach(e -> after.addTestMetric(id2, "metric-6", e)); 144 145 ArgumentCaptor<List<TableRow>> runResultCaptor = ArgumentCaptor.forClass(List.class); 146 ArgumentCaptor<MultiMap<String, TableRow>> testResultCaptor = 147 ArgumentCaptor.forClass(MultiMap.class); 148 detector.runRegressionDetection(before, after); 149 verify(detector, times(1)) 150 .logResult( 151 eq(before), 152 eq(after), 153 runResultCaptor.capture(), 154 testResultCaptor.capture()); 155 156 List<TableRow> runResults = runResultCaptor.getValue(); 157 assertEquals(1, runResults.size()); 158 assertEquals("metric-3", runResults.get(0).name); 159 160 MultiMap<String, TableRow> testResults = testResultCaptor.getValue(); 161 assertEquals(1, testResults.size()); 162 assertEquals(1, testResults.get(id2.toString()).size()); 163 assertEquals("metric-6", testResults.get(id2.toString()).get(0).name); 164 } 165 166 private boolean equal(double d1, double d2) { 167 return Math.abs(d1 - d2) < EPS; 168 } 169 } 170
