1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """This module contains unit tests for firmware_summary module.""" 6 7 8 import os 9 import unittest 10 11 import common_unittest_utils 12 13 from firmware_constants import VAL 14 from firmware_summary import FirmwareSummary 15 16 17 # Define the relative segment weights of a validator. 18 segment_weights = {VAL.BEGIN: 0.15, 19 VAL.MIDDLE: 0.7, 20 VAL.END: 0.15, 21 VAL.BOTH_ENDS: 0.15 + 0.15, 22 VAL.WHOLE: 0.15 + 0.7 + 0.15} 23 24 # Define the validator score weights 25 weight_rare = 1 26 weight_common = 2 27 weight_critical = 3 28 validator_weights = {'CountPacketsValidator': weight_common, 29 'CountTrackingIDFatFingerValidator': weight_rare, 30 'CountTrackingIDNormalFingerValidator': weight_critical, 31 'DrumrollValidator': weight_rare, 32 'LinearityFatFingerValidator': weight_rare, 33 'LinearityNormalFingerValidator': weight_common, 34 'NoGapValidator': weight_common, 35 'NoLevelJumpValidator': weight_rare, 36 'NoReversedMotionValidator': weight_common, 37 'PhysicalClickValidator': weight_critical, 38 'PinchValidator': weight_common, 39 'RangeValidator': weight_common, 40 'ReportRateValidator': weight_common, 41 'StationaryFingerValidator': weight_common, 42 'StationaryTapValidator': weight_common} 43 44 45 class FirmwareSummaryTest(unittest.TestCase): 46 """A base class for FirwareSummary unit tests.""" 47 @classmethod 48 def setUpClass(cls): 49 test_dir = os.path.join(os.getcwd(), 'tests') 50 log_dir = os.path.join(test_dir, 'logs', cls.log_category) 51 summary = FirmwareSummary(log_dir=log_dir, 52 validator_weights=validator_weights, 53 segment_weights=segment_weights) 54 cls.slog = summary.slog 55 cls._round_digits = 8 56 57 def _get_score(self, fw=None, gesture=None, validator=None): 58 """Score = sum / count, rounded to the 4th digit.""" 59 result= self.slog.get_result(fw=fw, gesture=gesture, 60 validators=validator) 61 average = result.stat_scores.average 62 return round(average, self._round_digits) 63 64 65 class FirmwareSummaryLumpyTest(FirmwareSummaryTest): 66 """Unit tests for firmware_summary.FirmwareSummary class using Lumpy logs. 67 68 Tests were conducted with both fw 11.23 and 11.26, and in combination of 69 single and multiple iterations. 70 """ 71 @classmethod 72 def setUpClass(cls): 73 cls.log_category = 'lumpy' 74 cls.fws = ['fw_11.23', 'fw_11.27'] 75 super(FirmwareSummaryLumpyTest, cls).setUpClass() 76 77 def _test_by_gesture(self, validator, expected_scores): 78 for fw, fw_expected_scores in expected_scores.items(): 79 for gesture, expected_score in fw_expected_scores.items(): 80 actual_score = self._get_score(fw=fw, 81 gesture=gesture, 82 validator=validator) 83 self.assertAlmostEqual(actual_score, expected_score) 84 85 def test_by_gesture_CountTrackingIDNormalFingerValidator(self): 86 validator = 'CountTrackingIDNormalFingerValidator' 87 expected_scores = { 88 'fw_11.23': { 89 'one_finger_tracking': 1.0, 90 'two_finger_tracking': 1.0, 91 }, 92 'fw_11.27': { 93 'one_finger_tracking': 1.0, 94 'two_finger_tracking': 1.0, 95 } 96 } 97 self._test_by_gesture(validator, expected_scores) 98 99 def test_by_gesture_CountTrackingIDFatFingerValidator(self): 100 validator = 'CountTrackingIDFatFingerValidator' 101 expected_scores = { 102 'fw_11.23': {'drag_edge_thumb': 0.0,}, 103 'fw_11.27': {'drag_edge_thumb': 0.5,} 104 } 105 self._test_by_gesture(validator, expected_scores) 106 107 def test_by_gesture_DrumrollValidator(self): 108 validator = 'DrumrollValidator' 109 expected_scores = { 110 'fw_11.23': { 111 'drumroll': 0.75, 112 }, 113 'fw_11.27': { 114 'drumroll': 0.66666667, 115 } 116 } 117 self._test_by_gesture(validator, expected_scores) 118 119 def test_by_gesture_LinearityMiddleValidator(self): 120 validator = 'LinearityNormalFinger(Middle)Validator' 121 expected_scores = { 122 'fw_11.23': { 123 'one_finger_to_edge': 0.58086671000000001, 124 'one_finger_tracking': 0.42046572999999998, 125 'two_finger_tracking': 0.60548126000000002, 126 }, 127 'fw_11.27': { 128 'one_finger_to_edge': 0.36506074999999999, 129 'one_finger_tracking': 0.73313022999999999, 130 'two_finger_tracking': 0.70906895999999997, 131 } 132 } 133 self._test_by_gesture(validator, expected_scores) 134 135 def test_by_gesture_NoGapValidator(self): 136 validator = 'NoGapValidator' 137 expected_scores = { 138 'fw_11.23': { 139 'one_finger_to_edge': 0.16022362, 140 'one_finger_tracking': 0.11006574, 141 'two_finger_tracking': 0.09455679, 142 }, 143 'fw_11.27': { 144 'one_finger_to_edge': 0.00000000, 145 'one_finger_tracking': 0.86488696, 146 'two_finger_tracking': 0.76206434, 147 } 148 } 149 self._test_by_gesture(validator, expected_scores) 150 151 def test_by_gesture_PhysicalClickValidator(self): 152 validator = 'PhysicalClickValidator' 153 expected_scores = { 154 'fw_11.23': { 155 'one_finger_physical_click': 0.875, 156 'two_fingers_physical_click': 0.25, 157 }, 158 'fw_11.27': { 159 'one_finger_physical_click': 1.0, 160 'two_fingers_physical_click': 1.0, 161 } 162 } 163 self._test_by_gesture(validator, expected_scores) 164 165 def test_by_gesture_StationaryTapValidator(self): 166 validator = 'StationaryTapValidator' 167 expected_scores = { 168 'fw_11.23': { 169 'one_finger_physical_click': 0.1875, 170 'two_fingers_physical_click': 0.125, 171 }, 172 'fw_11.27': { 173 'one_finger_physical_click': 0.58333332999999998, 174 'two_fingers_physical_click': 0.16666666999999999, 175 } 176 } 177 self._test_by_gesture(validator, expected_scores) 178 179 def test_by_validator(self): 180 expected_scores = { 181 'fw_11.23': { 182 'CountPacketsValidator': 0.895833333333, 183 'CountTrackingIDFatFingerValidator': 0.0, 184 'CountTrackingIDNormalFingerValidator': 0.96666666999999995, 185 'DrumrollValidator': 0.75, 186 'LinearityNormalFinger(Middle)Validator': 0.54910331999999995, 187 'NoGapValidator': 0.101144302433, 188 'PhysicalClickValidator': 0.75, 189 'PinchValidator': 0.875, 190 'StationaryTapValidator': 0.16666666999999999, 191 }, 192 'fw_11.27': { 193 'CountPacketsValidator': 1.0, 194 'CountTrackingIDFatFingerValidator': 0.5, 195 'CountTrackingIDNormalFingerValidator': 0.95555555999999997, 196 'DrumrollValidator': 0.666666666667, 197 'LinearityNormalFinger(Middle)Validator': 0.66679957999999995, 198 'NoGapValidator': 0.623221473233, 199 'PhysicalClickValidator': 1.0, 200 'PinchValidator': 1.0, 201 'StationaryTapValidator': 0.44444444, 202 } 203 } 204 for fw, fw_expected_scores in expected_scores.items(): 205 for validator, expected_score in fw_expected_scores.items(): 206 actual_score = self._get_score(fw=fw, validator=validator) 207 actual_score = round(actual_score, self._round_digits) 208 self.assertAlmostEqual(actual_score, expected_score) 209 210 def test_stat_metrics(self): 211 """Test the statistics of metrics.""" 212 expected_stats_values = { 213 'fw_11.23': { 214 'CountPacketsValidator': [ 215 ('pct of incorrect cases (%)--packets', 25.00)], 216 'PhysicalClickValidator': [ 217 ('1f-click miss rate (%)', 12.50), 218 ('2f-click miss rate (%)', 75.00)], 219 'PinchValidator': [ 220 ('pct of incorrect cases (%)--pinch', 12.50)], 221 }, 222 'fw_11.27': { 223 'CountPacketsValidator': [ 224 ('pct of incorrect cases (%)--packets', 0.00)], 225 'PhysicalClickValidator': [ 226 ('1f-click miss rate (%)', 0.00), 227 ('2f-click miss rate (%)', 0.00)], 228 'PinchValidator': [ 229 ('pct of incorrect cases (%)--pinch', 0.00)], 230 }, 231 } 232 233 for fw, fw_stats_values in expected_stats_values.items(): 234 for validator, stats_metrics in fw_stats_values.items(): 235 result = self.slog.get_result(fw=fw, validators=validator) 236 for metric_name, expected_value in stats_metrics: 237 actual_value = result.stat_metrics.stats_values[metric_name] 238 self.assertAlmostEqual(actual_value, expected_value) 239 240 def test_final_weighted_average(self): 241 expected_weighted_averages = { 242 'fw_11.23': 0.68406327, 243 'fw_11.27': 0.83886367, 244 } 245 final_weighted_average = self.slog.get_final_weighted_average() 246 for fw, expected_value in expected_weighted_averages.items(): 247 actual_value = final_weighted_average[fw] 248 actual_value = round(actual_value, self._round_digits) 249 self.assertAlmostEqual(actual_value, expected_value) 250 251 252 if __name__ == '__main__': 253 unittest.main() 254
