1 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 import time 8 9 from autotest_lib.client.bin import test, utils 10 from autotest_lib.client.common_lib import error 11 from autotest_lib.client.common_lib import file_utils 12 from autotest_lib.client.common_lib.cros import chrome 13 from autotest_lib.client.cros import service_stopper 14 from autotest_lib.client.cros.power import power_status, power_utils 15 from autotest_lib.client.cros.video import histogram_verifier 16 from autotest_lib.client.cros.video import constants 17 from autotest_lib.client.cros.video import helper_logger 18 19 20 EXTRA_BROWSER_ARGS = ['--use-fake-device-for-media-stream', 21 '--use-fake-ui-for-media-stream'] 22 FAKE_FILE_ARG = '--use-file-for-fake-video-capture="%s"' 23 DISABLE_ACCELERATED_VIDEO_DECODE_BROWSER_ARGS = [ 24 '--disable-accelerated-video-decode'] 25 26 DOWNLOAD_BASE = ('http://commondatastorage.googleapis.com/' 27 'chromiumos-test-assets-public/crowd/') 28 VIDEO_NAME = 'crowd720_25frames.y4m' 29 30 WEBRTC_INTERNALS_URL = 'chrome://webrtc-internals' 31 32 WEBRTC_WITH_HW_ACCELERATION = 'webrtc_with_hw_acceleration' 33 WEBRTC_WITHOUT_HW_ACCELERATION = 'webrtc_without_hw_acceleration' 34 35 # Measurement duration in seconds. 36 MEASUREMENT_DURATION = 30 37 # Time to exclude from calculation after start the loopback [seconds]. 38 STABILIZATION_DURATION = 10 39 40 # Time in seconds to wait for cpu idle until giving up. 41 WAIT_FOR_IDLE_CPU_TIMEOUT = 60.0 42 # Maximum percent of cpu usage considered as idle. 43 CPU_IDLE_USAGE = 0.1 44 45 MAX_DECODE_TIME_DESCRIPTION = 'decode_time.max' 46 MEDIAN_DECODE_TIME_DESCRIPTION = 'decode_time.percentile_0.50' 47 CPU_USAGE_DESCRIPTION = 'video_cpu_usage' 48 POWER_DESCRIPTION = 'video_mean_energy_rate' 49 50 # Minimum battery charge percentage to run the power test. 51 BATTERY_INITIAL_CHARGED_MIN = 10 52 53 # These are the variable names in WebRTC internals. 54 # Maximum decode time of the frames of the last 10 seconds. 55 GOOG_MAX_DECODE_MS = 'googMaxDecodeMs' 56 # The decode time of the last frame. 57 GOOG_DECODE_MS = 'googDecodeMs' 58 59 # Javascript function to get the decode time. addStats is a function called by 60 # Chrome to pass WebRTC statistics every second. 61 ADD_STATS_JAVASCRIPT = ( 62 'var googMaxDecodeMs = new Array();' 63 'var googDecodeMs = new Array();' 64 'addStats = function(data) {' 65 ' reports = data.reports;' 66 ' for (var i=0; i < reports.length; i++) {' 67 ' if (reports[i].type == "ssrc") {' 68 ' values = reports[i].stats.values;' 69 ' for (var j=0; j < values.length; j++) {' 70 ' if (values[j] == "googMaxDecodeMs")' 71 ' googMaxDecodeMs[googMaxDecodeMs.length] = values[j+1];' 72 ' else if (values[j] == "googDecodeMs")' 73 ' googDecodeMs[googDecodeMs.length] = values[j+1];' 74 ' }' 75 ' }' 76 ' }' 77 '}') 78 79 # Measure the stats until getting 10 samples or exceeding 30 seconds. addStats 80 # is called once per second for now. 81 NUM_DECODE_TIME_SAMPLES = 10 82 TIMEOUT = 60 83 84 85 class video_WebRtcPerf(test.test): 86 """ 87 The test outputs the decode time, cpu usage and the power consumption for 88 WebRTC to performance dashboard. 89 """ 90 version = 1 91 arc_mode = None 92 93 94 def initialize(self): 95 self._service_stopper = None 96 self._original_governors = None 97 self._backlight = None 98 99 100 def cleanup(self): 101 if self._backlight: 102 self._backlight.restore() 103 if self._service_stopper: 104 self._service_stopper.restore_services() 105 if self._original_governors: 106 utils.restore_scaling_governor_states(self._original_governors) 107 108 super(video_WebRtcPerf, self).cleanup() 109 110 111 def start_loopback(self, cr): 112 """ 113 Opens WebRTC loopback page. 114 115 @param cr: Autotest Chrome instance. 116 """ 117 cr.browser.platform.SetHTTPServerDirectories(self.bindir) 118 119 tab = cr.browser.tabs[0] 120 tab.Navigate(cr.browser.platform.http_server.UrlOf( 121 os.path.join(self.bindir, 'loopback.html'))) 122 tab.WaitForDocumentReadyStateToBeComplete() 123 124 125 def open_stats_page(self, cr): 126 """ 127 Opens WebRTC internal statistics page. 128 129 @param cr: Autotest Chrome instance. 130 131 @returns the tab of the stats page. 132 """ 133 tab = cr.browser.tabs.New() 134 tab.Navigate(WEBRTC_INTERNALS_URL) 135 tab.WaitForDocumentReadyStateToBeComplete() 136 # Inject stats callback function. 137 tab.EvaluateJavaScript(ADD_STATS_JAVASCRIPT) 138 return tab 139 140 141 @helper_logger.video_log_wrapper 142 def run_once(self, decode_time_test=False, cpu_test=False, 143 power_test=False, arc_mode=None): 144 """ 145 Runs the video_WebRtcPerf test. 146 147 @param decode_time_test: Pass True to run decode time test. 148 @param cpu_test: Pass True to run CPU usage test. 149 @param power_test: Pass True to run power consumption test. 150 @param arc_mode: if 'enabled', run the test with Android enabled. 151 """ 152 # Download test video. 153 url = DOWNLOAD_BASE + VIDEO_NAME 154 local_path = os.path.join(self.bindir, VIDEO_NAME) 155 file_utils.download_file(url, local_path) 156 self.arc_mode = arc_mode 157 158 if decode_time_test: 159 keyvals = self.test_decode_time(local_path) 160 # The first value is max decode time. The second value is median 161 # decode time. Construct a dictionary containing one value to log 162 # the result. 163 max_decode_time = { 164 key:value[0] for (key, value) in keyvals.items()} 165 self.log_result(max_decode_time, MAX_DECODE_TIME_DESCRIPTION, 166 'milliseconds') 167 median_decode_time = { 168 key:value[1] for (key, value) in keyvals.items()} 169 self.log_result(median_decode_time, MEDIAN_DECODE_TIME_DESCRIPTION, 170 'milliseconds') 171 172 if cpu_test: 173 keyvals = self.test_cpu_usage(local_path) 174 self.log_result(keyvals, CPU_USAGE_DESCRIPTION, 'percent') 175 176 if power_test: 177 keyvals = self.test_power(local_path) 178 self.log_result(keyvals, POWER_DESCRIPTION , 'W') 179 180 181 def test_webrtc(self, local_path, gather_result): 182 """ 183 Runs the webrtc test with and without hardware acceleration. 184 185 @param local_path: the path to the video file. 186 @param gather_result: a function to run and return the test result 187 after chrome opens. The input parameter of the funciton is 188 Autotest chrome instance. 189 190 @return a dictionary that contains test the result. 191 """ 192 keyvals = {} 193 EXTRA_BROWSER_ARGS.append(FAKE_FILE_ARG % local_path) 194 195 with chrome.Chrome(extra_browser_args=EXTRA_BROWSER_ARGS +\ 196 [helper_logger.chrome_vmodule_flag()], 197 arc_mode=self.arc_mode, 198 init_network_controller=True) as cr: 199 # On daisy, Chrome freezes about 30 seconds after login because of 200 # TPM error. See http://crbug.com/588579. 201 if utils.get_board() == 'daisy': 202 logging.warning('Delay 30s for issue 588579 on daisy') 203 time.sleep(30) 204 # Open WebRTC loopback page and start the loopback. 205 self.start_loopback(cr) 206 result = gather_result(cr) 207 208 # Check if decode is hardware accelerated. 209 if histogram_verifier.is_bucket_present( 210 cr, 211 constants.RTC_INIT_HISTOGRAM, 212 constants.RTC_VIDEO_INIT_BUCKET): 213 keyvals[WEBRTC_WITH_HW_ACCELERATION] = result 214 else: 215 logging.info("Can not use hardware decoding.") 216 keyvals[WEBRTC_WITHOUT_HW_ACCELERATION] = result 217 return keyvals 218 219 # Start chrome with disabled video hardware decode flag. 220 with chrome.Chrome(extra_browser_args= 221 DISABLE_ACCELERATED_VIDEO_DECODE_BROWSER_ARGS + 222 EXTRA_BROWSER_ARGS, arc_mode=self.arc_mode, 223 init_network_controller=True) as cr: 224 225 # crbug/753292 - enforce the idle checks after login 226 if not utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, 227 CPU_IDLE_USAGE): 228 logging.warning('Could not get idle CPU post login.') 229 if not utils.wait_for_cool_machine(): 230 logging.warning('Could not get cold machine post login.') 231 232 if utils.get_board() == 'daisy': 233 logging.warning('Delay 30s for issue 588579 on daisy') 234 time.sleep(30) 235 # Open the webrtc loopback page and start the loopback. 236 self.start_loopback(cr) 237 result = gather_result(cr) 238 239 # Make sure decode is not hardware accelerated. 240 if histogram_verifier.is_bucket_present( 241 cr, 242 constants.RTC_INIT_HISTOGRAM, 243 constants.RTC_VIDEO_INIT_BUCKET): 244 raise error.TestError('HW decode should not be used.') 245 keyvals[WEBRTC_WITHOUT_HW_ACCELERATION] = result 246 247 return keyvals 248 249 250 def test_cpu_usage(self, local_path): 251 """ 252 Runs the video cpu usage test. 253 254 @param local_path: the path to the video file. 255 256 @return a dictionary that contains the test result. 257 """ 258 def get_cpu_usage(cr): 259 time.sleep(STABILIZATION_DURATION) 260 cpu_usage_start = utils.get_cpu_usage() 261 time.sleep(MEASUREMENT_DURATION) 262 cpu_usage_end = utils.get_cpu_usage() 263 return utils.compute_active_cpu_time(cpu_usage_start, 264 cpu_usage_end) * 100 265 266 # crbug/753292 - APNG login pictures increase CPU usage. Move the more 267 # strict idle checks after the login phase. 268 utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, CPU_IDLE_USAGE) 269 utils.wait_for_cool_machine() 270 if not utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, 271 CPU_IDLE_USAGE): 272 logging.warning('Could not get idle CPU pre login.') 273 if not utils.wait_for_cool_machine(): 274 logging.warning('Could not get cold machine pre login.') 275 276 # Stop the thermal service that may change the cpu frequency. 277 self._service_stopper = service_stopper.get_thermal_service_stopper() 278 self._service_stopper.stop_services() 279 # Set the scaling governor to performance mode to set the cpu to the 280 # highest frequency available. 281 self._original_governors = utils.set_high_performance_mode() 282 return self.test_webrtc(local_path, get_cpu_usage) 283 284 285 def test_power(self, local_path): 286 """ 287 Runs the video power consumption test. 288 289 @param local_path: the path to the video file. 290 291 @return a dictionary that contains the test result. 292 """ 293 self._backlight = power_utils.Backlight() 294 self._backlight.set_default() 295 self._service_stopper = service_stopper.ServiceStopper( 296 service_stopper.ServiceStopper.POWER_DRAW_SERVICES) 297 self._service_stopper.stop_services() 298 299 current_power_status = power_status.get_status() 300 # We expect the DUT is powered by battery now. But this is not always 301 # true due to other bugs. Disable this test temporarily as workaround. 302 # TODO(kcwu): remove this workaround after AC control is stable 303 # crbug.com/723968 304 if current_power_status.on_ac(): 305 logging.warning('Still powered by AC. Skip this test') 306 return {} 307 # Verify that the battery is sufficiently charged. 308 current_power_status.assert_battery_state(BATTERY_INITIAL_CHARGED_MIN) 309 310 measurements = [power_status.SystemPower( 311 current_power_status.battery_path)] 312 313 def get_power(cr): 314 power_logger = power_status.PowerLogger(measurements) 315 power_logger.start() 316 time.sleep(STABILIZATION_DURATION) 317 start_time = time.time() 318 time.sleep(MEASUREMENT_DURATION) 319 power_logger.checkpoint('result', start_time) 320 keyval = power_logger.calc() 321 # save_results() will save <fname_prefix>_raw.txt and 322 # <fname_prefix>_summary.txt, where the former contains raw data. 323 fname_prefix = 'result_%.0f' % time.time() 324 power_logger.save_results(self.resultsdir, fname_prefix) 325 metric_name = 'result_' + measurements[0].domain 326 with open(os.path.join( 327 self.resultsdir, fname_prefix + '_raw.txt')) as f: 328 for line in f.readlines(): 329 if line.startswith(metric_name): 330 split_data = line.split('\t') 331 # split_data[0] is metric_name, [1:] are raw data. 332 return [float(data) for data in split_data[1:]] 333 # Return a list contains the average power only for fallback. 334 return [keyval[metric_name + '_pwr_avg']] 335 336 return self.test_webrtc(local_path, get_power) 337 338 339 def test_decode_time(self, local_path): 340 """ 341 Runs the decode time test. 342 343 @param local_path: the path to the video file. 344 345 @return a dictionary that contains the test result. 346 """ 347 def get_decode_time(cr): 348 tab = self.open_stats_page(cr) 349 # Collect the decode time until there are enough samples. 350 start_time = time.time() 351 max_decode_time_list = [] 352 decode_time_list = [] 353 while (time.time() - start_time < TIMEOUT and 354 len(decode_time_list) < NUM_DECODE_TIME_SAMPLES): 355 time.sleep(1) 356 max_decode_time_list = [] 357 decode_time_list = [] 358 try: 359 max_decode_time_list = [int(x) for x in 360 tab.EvaluateJavaScript(GOOG_MAX_DECODE_MS)] 361 decode_time_list = [int(x) for x in 362 tab.EvaluateJavaScript(GOOG_DECODE_MS)] 363 except: 364 pass 365 # Output the values if they are valid. 366 if len(max_decode_time_list) < NUM_DECODE_TIME_SAMPLES: 367 raise error.TestError('Not enough ' + GOOG_MAX_DECODE_MS) 368 if len(decode_time_list) < NUM_DECODE_TIME_SAMPLES: 369 raise error.TestError('Not enough ' + GOOG_DECODE_MS) 370 max_decode_time = max(max_decode_time_list) 371 decode_time_median = self.get_median(decode_time_list) 372 logging.info("Max decode time list=%s", str(max_decode_time_list)) 373 logging.info("Decode time list=%s", str(decode_time_list)) 374 logging.info("Maximum decode time=%d, median=%d", max_decode_time, 375 decode_time_median) 376 return (max_decode_time, decode_time_median) 377 378 return self.test_webrtc(local_path, get_decode_time) 379 380 381 def get_median(self, seq): 382 """ 383 Calculates the median of a sequence of numbers. 384 385 @param seq: a list with numbers. 386 387 @returns the median of the numbers. 388 """ 389 seq.sort() 390 size = len(seq) 391 if size % 2 != 0: 392 return seq[size / 2] 393 return (seq[size / 2] + seq[size / 2 - 1]) / 2.0 394 395 def log_result(self, keyvals, description, units): 396 """ 397 Logs the test result output to the performance dashboard. 398 399 @param keyvals: a dictionary that contains results returned by 400 test_webrtc. 401 @param description: a string that describes the video and test result 402 and it will be part of the entry name in the dashboard. 403 @param units: the units of test result. 404 """ 405 result_with_hw = keyvals.get(WEBRTC_WITH_HW_ACCELERATION) 406 if result_with_hw: 407 self.output_perf_value( 408 description= 'hw_' + description, value=result_with_hw, 409 units=units, higher_is_better=False) 410 411 result_without_hw = keyvals.get(WEBRTC_WITHOUT_HW_ACCELERATION) 412 if result_without_hw: 413 self.output_perf_value( 414 description= 'sw_' + description, value=result_without_hw, 415 units=units, higher_is_better=False) 416
