1 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 import time 8 9 from autotest_lib.client.bin import site_utils, test, utils 10 from autotest_lib.client.common_lib import error 11 from autotest_lib.client.common_lib import file_utils 12 from autotest_lib.client.common_lib.cros import chrome 13 from autotest_lib.client.cros import power_status, power_utils 14 from autotest_lib.client.cros import service_stopper 15 from autotest_lib.client.cros.video import histogram_verifier 16 from autotest_lib.client.cros.video import constants 17 18 19 EXTRA_BROWSER_ARGS = ['--use-fake-device-for-media-stream', 20 '--use-fake-ui-for-media-stream'] 21 FAKE_FILE_ARG = '--use-file-for-fake-video-capture="%s"' 22 DISABLE_ACCELERATED_VIDEO_DECODE_BROWSER_ARGS = [ 23 '--disable-accelerated-video-decode'] 24 25 DOWNLOAD_BASE = ('http://commondatastorage.googleapis.com/' 26 'chromiumos-test-assets-public/crowd/') 27 VIDEO_NAME = 'crowd720_25frames.y4m' 28 29 WEBRTC_INTERNALS_URL = 'chrome://webrtc-internals' 30 31 WEBRTC_WITH_HW_ACCELERATION = 'webrtc_with_hw_acceleration' 32 WEBRTC_WITHOUT_HW_ACCELERATION = 'webrtc_without_hw_acceleration' 33 34 # Measurement duration in seconds. 35 MEASUREMENT_DURATION = 30 36 # Time to exclude from calculation after start the loopback [seconds]. 37 STABILIZATION_DURATION = 10 38 39 # List of thermal throttling services that should be disabled. 40 # - temp_metrics for link. 41 # - thermal for daisy, snow, pit etc. 42 THERMAL_SERVICES = ['temp_metrics', 'thermal'] 43 44 # Time in seconds to wait for cpu idle until giving up. 45 WAIT_FOR_IDLE_CPU_TIMEOUT = 60.0 46 # Maximum percent of cpu usage considered as idle. 47 CPU_IDLE_USAGE = 0.1 48 49 MAX_DECODE_TIME_DESCRIPTION = 'decode_time.max' 50 MEDIAN_DECODE_TIME_DESCRIPTION = 'decode_time.percentile_0.50' 51 CPU_USAGE_DESCRIPTION = 'video_cpu_usage' 52 POWER_DESCRIPTION = 'video_mean_energy_rate' 53 54 # Minimum battery charge percentage to run the power test. 55 BATTERY_INITIAL_CHARGED_MIN = 10 56 57 # These are the variable names in WebRTC internals. 58 # Maximum decode time of the frames of the last 10 seconds. 59 GOOG_MAX_DECODE_MS = 'googMaxDecodeMs' 60 # The decode time of the last frame. 61 GOOG_DECODE_MS = 'googDecodeMs' 62 63 # Javascript function to get the decode time. addStats is a function called by 64 # Chrome to pass WebRTC statistics every second. 65 ADD_STATS_JAVASCRIPT = ( 66 'var googMaxDecodeMs = new Array();' 67 'var googDecodeMs = new Array();' 68 'addStats = function(data) {' 69 ' reports = data.reports;' 70 ' for (var i=0; i < reports.length; i++) {' 71 ' if (reports[i].type == "ssrc") {' 72 ' values = reports[i].stats.values;' 73 ' for (var j=0; j < values.length; j++) {' 74 ' if (values[j] == "googMaxDecodeMs")' 75 ' googMaxDecodeMs[googMaxDecodeMs.length] = values[j+1];' 76 ' else if (values[j] == "googDecodeMs")' 77 ' googDecodeMs[googDecodeMs.length] = values[j+1];' 78 ' }' 79 ' }' 80 ' }' 81 '}') 82 83 # Measure the stats until getting 10 samples or exceeding 15 seconds. addStats 84 # is called once per second for now. 85 NUM_DECODE_TIME_SAMPLES = 10 86 TIMEOUT = 15 87 88 89 class video_WebRtcPerf(test.test): 90 """ 91 The test outputs the decode time, cpu usage and the power consumption for 92 WebRTC to performance dashboard. 93 """ 94 version = 1 95 96 97 def start_loopback(self, cr): 98 """ 99 Opens WebRTC loopback page. 100 101 @param cr: Autotest Chrome instance. 102 """ 103 cr.browser.platform.SetHTTPServerDirectories(self.bindir) 104 105 tab = cr.browser.tabs[0] 106 tab.Navigate(cr.browser.platform.http_server.UrlOf( 107 os.path.join(self.bindir, 'loopback.html'))) 108 tab.WaitForDocumentReadyStateToBeComplete() 109 110 111 def open_stats_page(self, cr): 112 """ 113 Opens WebRTC internal statistics page. 114 115 @param cr: Autotest Chrome instance. 116 117 @returns the tab of the stats page. 118 """ 119 tab = cr.browser.tabs.New() 120 tab.Navigate(WEBRTC_INTERNALS_URL) 121 tab.WaitForDocumentReadyStateToBeComplete() 122 # Inject stats callback function. 123 tab.EvaluateJavaScript(ADD_STATS_JAVASCRIPT) 124 return tab 125 126 127 def run_once(self, decode_time_test=False, cpu_test=False, power_test=False): 128 """ 129 Runs the video_WebRtcPerf test. 130 131 @param decode_time_test: Pass True to run decode time test. 132 @param cpu_test: Pass True to run CPU usage test. 133 @param power_test: Pass True to run power consumption test. 134 """ 135 # Download test video. 136 url = DOWNLOAD_BASE + VIDEO_NAME 137 local_path = os.path.join(self.bindir, VIDEO_NAME) 138 file_utils.download_file(url, local_path) 139 140 if decode_time_test: 141 keyvals = self.test_decode_time(local_path) 142 # The first value is max decode time. The second value is median 143 # decode time. Construct a dictionary containing one value to log 144 # the result. 145 max_decode_time = { 146 key:value[0] for (key, value) in keyvals.items()} 147 self.log_result(max_decode_time, MAX_DECODE_TIME_DESCRIPTION, 148 'milliseconds') 149 median_decode_time = { 150 key:value[1] for (key, value) in keyvals.items()} 151 self.log_result(median_decode_time, MEDIAN_DECODE_TIME_DESCRIPTION, 152 'milliseconds') 153 154 if cpu_test: 155 keyvals = self.test_cpu_usage(local_path) 156 self.log_result(keyvals, CPU_USAGE_DESCRIPTION, 'percent') 157 158 if power_test: 159 keyvals = self.test_power(local_path) 160 self.log_result(keyvals, POWER_DESCRIPTION , 'W') 161 162 163 def test_webrtc(self, local_path, gather_result): 164 """ 165 Runs the webrtc test with and without hardware acceleration. 166 167 @param local_path: the path to the video file. 168 @param gather_result: a function to run and return the test result 169 after chrome opens. The input parameter of the funciton is 170 Autotest chrome instance. 171 172 @return a dictionary that contains test the result. 173 """ 174 keyvals = {} 175 EXTRA_BROWSER_ARGS.append(FAKE_FILE_ARG % local_path) 176 177 with chrome.Chrome(extra_browser_args=EXTRA_BROWSER_ARGS) as cr: 178 # Open WebRTC loopback page and start the loopback. 179 self.start_loopback(cr) 180 result = gather_result(cr) 181 182 # Check if decode is hardware accelerated. 183 if histogram_verifier.is_bucket_present( 184 cr, 185 constants.RTC_INIT_HISTOGRAM, 186 constants.RTC_VIDEO_INIT_BUCKET): 187 keyvals[WEBRTC_WITH_HW_ACCELERATION] = result 188 else: 189 logging.info("Can not use hardware decoding.") 190 keyvals[WEBRTC_WITHOUT_HW_ACCELERATION] = result 191 return keyvals 192 193 # Start chrome with disabled video hardware decode flag. 194 with chrome.Chrome(extra_browser_args= 195 DISABLE_ACCELERATED_VIDEO_DECODE_BROWSER_ARGS + 196 EXTRA_BROWSER_ARGS) as cr: 197 # Open the webrtc loopback page and start the loopback. 198 self.start_loopback(cr) 199 result = gather_result(cr) 200 201 # Make sure decode is not hardware accelerated. 202 if histogram_verifier.is_bucket_present( 203 cr, 204 constants.RTC_INIT_HISTOGRAM, 205 constants.RTC_VIDEO_INIT_BUCKET): 206 raise error.TestError('HW decode should not be used.') 207 keyvals[WEBRTC_WITHOUT_HW_ACCELERATION] = result 208 209 return keyvals 210 211 212 def test_cpu_usage(self, local_path): 213 """ 214 Runs the video cpu usage test. 215 216 @param local_path: the path to the video file. 217 218 @return a dictionary that contains the test result. 219 """ 220 def get_cpu_usage(cr): 221 time.sleep(STABILIZATION_DURATION) 222 cpu_usage_start = site_utils.get_cpu_usage() 223 time.sleep(MEASUREMENT_DURATION) 224 cpu_usage_end = site_utils.get_cpu_usage() 225 return site_utils.compute_active_cpu_time(cpu_usage_start, 226 cpu_usage_end) * 100 227 if not utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, 228 CPU_IDLE_USAGE): 229 raise error.TestError('Could not get idle CPU.') 230 if not utils.wait_for_cool_machine(): 231 raise error.TestError('Could not get cool machine.') 232 # Stop the thermal service that may change the cpu frequency. 233 services = service_stopper.ServiceStopper(THERMAL_SERVICES) 234 services.stop_services() 235 # Set the scaling governor to performance mode to set the cpu to the 236 # highest frequency available. 237 original_governors = utils.set_high_performance_mode() 238 try: 239 return self.test_webrtc(local_path, get_cpu_usage) 240 finally: 241 services.restore_services() 242 utils.restore_scaling_governor_states(original_governors) 243 244 245 def test_power(self, local_path): 246 """ 247 Runs the video power consumption test. 248 249 @param local_path: the path to the video file. 250 251 @return a dictionary that contains the test result. 252 """ 253 # Verify that we are running on battery and the battery is sufficiently 254 # charged. 255 current_power_status = power_status.get_status() 256 current_power_status.assert_battery_state(BATTERY_INITIAL_CHARGED_MIN) 257 258 measurements = [power_status.SystemPower( 259 current_power_status.battery_path)] 260 261 def get_power(cr): 262 power_logger = power_status.PowerLogger(measurements) 263 power_logger.start() 264 time.sleep(STABILIZATION_DURATION) 265 start_time = time.time() 266 time.sleep(MEASUREMENT_DURATION) 267 power_logger.checkpoint('result', start_time) 268 keyval = power_logger.calc() 269 return keyval['result_' + measurements[0].domain + '_pwr'] 270 271 backlight = power_utils.Backlight() 272 backlight.set_default() 273 services = service_stopper.ServiceStopper( 274 service_stopper.ServiceStopper.POWER_DRAW_SERVICES) 275 services.stop_services() 276 try: 277 return self.test_webrtc(local_path, get_power) 278 finally: 279 backlight.restore() 280 services.restore_services() 281 282 283 def test_decode_time(self, local_path): 284 """ 285 Runs the decode time test. 286 287 @param local_path: the path to the video file. 288 289 @return a dictionary that contains the test result. 290 """ 291 def get_decode_time(cr): 292 tab = self.open_stats_page(cr) 293 # Collect the decode time until there are enough samples. 294 start_time = time.time() 295 max_decode_time_list = [] 296 decode_time_list = [] 297 while (time.time() - start_time < TIMEOUT and 298 len(decode_time_list) < NUM_DECODE_TIME_SAMPLES): 299 time.sleep(1) 300 max_decode_time_list = [] 301 decode_time_list = [] 302 try: 303 max_decode_time_list = [int(x) for x in 304 tab.EvaluateJavaScript(GOOG_MAX_DECODE_MS)] 305 decode_time_list = [int(x) for x in 306 tab.EvaluateJavaScript(GOOG_DECODE_MS)] 307 except: 308 pass 309 # Output the values if they are valid. 310 if len(max_decode_time_list) < NUM_DECODE_TIME_SAMPLES: 311 raise error.TestError('Not enough ' + GOOG_MAX_DECODE_MS) 312 if len(decode_time_list) < NUM_DECODE_TIME_SAMPLES: 313 raise error.TestError('Not enough ' + GOOG_DECODE_MS) 314 max_decode_time = max(max_decode_time_list) 315 decode_time_median = self.get_median(decode_time_list) 316 logging.info("Max decode time list=%s", str(max_decode_time_list)) 317 logging.info("Decode time list=%s", str(decode_time_list)) 318 logging.info("Maximum decode time=%d, median=%d", max_decode_time, 319 decode_time_median) 320 return (max_decode_time, decode_time_median) 321 322 return self.test_webrtc(local_path, get_decode_time) 323 324 325 def get_median(self, seq): 326 """ 327 Calculates the median of a sequence of numbers. 328 329 @param seq: a list with numbers. 330 331 @returns the median of the numbers. 332 """ 333 seq.sort() 334 size = len(seq) 335 if size % 2 != 0: 336 return seq[size / 2] 337 return (seq[size / 2] + seq[size / 2 - 1]) / 2.0 338 339 def log_result(self, keyvals, description, units): 340 """ 341 Logs the test result output to the performance dashboard. 342 343 @param keyvals: a dictionary that contains results returned by 344 test_webrtc. 345 @param description: a string that describes the video and test result 346 and it will be part of the entry name in the dashboard. 347 @param units: the units of test result. 348 """ 349 result_with_hw = keyvals.get(WEBRTC_WITH_HW_ACCELERATION) 350 if result_with_hw: 351 self.output_perf_value( 352 description= 'hw_' + description, value=result_with_hw, 353 units=units, higher_is_better=False) 354 355 result_without_hw = keyvals.get(WEBRTC_WITHOUT_HW_ACCELERATION) 356 if result_without_hw: 357 self.output_perf_value( 358 description= 'sw_' + description, value=result_without_hw, 359 units=units, higher_is_better=False) 360
