1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """This is a client side WebGL aquarium test. 6 7 Description of some of the test result output: 8 - interframe time: The time elapsed between two frames. It is the elapsed 9 time between two consecutive calls to the render() function. 10 - render time: The time it takes in Javascript to construct a frame and 11 submit all the GL commands. It is the time it takes for a render() 12 function call to complete. 13 """ 14 15 import functools 16 import logging 17 import math 18 import os 19 import sampler 20 import system_sampler 21 import threading 22 import time 23 24 from autotest_lib.client.bin import fps_meter 25 from autotest_lib.client.bin import utils 26 from autotest_lib.client.common_lib import error 27 from autotest_lib.client.common_lib.cros import chrome 28 from autotest_lib.client.common_lib.cros import memory_eater 29 from autotest_lib.client.cros.graphics import graphics_utils 30 from autotest_lib.client.cros import service_stopper 31 from autotest_lib.client.cros.power import power_rapl, power_status, power_utils 32 33 # Minimum battery charge percentage to run the test 34 BATTERY_INITIAL_CHARGED_MIN = 10 35 36 # Measurement duration in seconds. 37 MEASUREMENT_DURATION = 30 38 39 POWER_DESCRIPTION = 'avg_energy_rate_1000_fishes' 40 41 # Time to exclude from calculation after playing a webgl demo [seconds]. 42 STABILIZATION_DURATION = 10 43 44 45 class graphics_WebGLAquarium(graphics_utils.GraphicsTest): 46 """WebGL aquarium graphics test.""" 47 version = 1 48 49 _backlight = None 50 _power_status = None 51 _service_stopper = None 52 _test_power = False 53 active_tab = None 54 flip_stats = {} 55 kernel_sampler = None 56 perf_keyval = {} 57 sampler_lock = None 58 test_duration_secs = 30 59 test_setting_num_fishes = 50 60 test_settings = { 61 50: ('setSetting2', 2), 62 1000: ('setSetting6', 6), 63 } 64 65 def setup(self): 66 """Testcase setup.""" 67 tarball_path = os.path.join(self.bindir, 68 'webgl_aquarium_static.tar.bz2') 69 utils.extract_tarball_to_dir(tarball_path, self.srcdir) 70 71 def initialize(self): 72 """Testcase initialization.""" 73 super(graphics_WebGLAquarium, self).initialize() 74 self.sampler_lock = threading.Lock() 75 # TODO: Create samplers for other platforms (e.g. x86). 76 if utils.get_board().lower() in ['daisy', 'daisy_spring']: 77 # Enable ExynosSampler on Exynos platforms. The sampler looks for 78 # exynos-drm page flip states: 'wait_kds', 'rendered', 'prepared', 79 # and 'flipped' in kernel debugfs. 80 81 # Sample 3-second durtaion for every 5 seconds. 82 self.kernel_sampler = sampler.ExynosSampler(period=5, duration=3) 83 self.kernel_sampler.sampler_callback = self.exynos_sampler_callback 84 self.kernel_sampler.output_flip_stats = ( 85 self.exynos_output_flip_stats) 86 87 def cleanup(self): 88 """Testcase cleanup.""" 89 if self._backlight: 90 self._backlight.restore() 91 if self._service_stopper: 92 self._service_stopper.restore_services() 93 super(graphics_WebGLAquarium, self).cleanup() 94 95 def setup_webpage(self, browser, test_url, num_fishes): 96 """Open fish tank in a new tab. 97 98 @param browser: The Browser object to run the test with. 99 @param test_url: The URL to the aquarium test site. 100 @param num_fishes: The number of fishes to run the test with. 101 """ 102 # Create tab and load page. Set the number of fishes when page is fully 103 # loaded. 104 tab = browser.tabs.New() 105 tab.Navigate(test_url) 106 tab.Activate() 107 self.active_tab = tab 108 tab.WaitForDocumentReadyStateToBeComplete() 109 110 # Set the number of fishes when document finishes loading. Also reset 111 # our own FPS counter and start recording FPS and rendering time. 112 utils.wait_for_value( 113 lambda: tab.EvaluateJavaScript( 114 'if (document.readyState === "complete") {' 115 ' setSetting(document.getElementById("%s"), %d);' 116 ' g_crosFpsCounter.reset();' 117 ' true;' 118 '} else {' 119 ' false;' 120 '}' % self.test_settings[num_fishes] 121 ), 122 expected_value=True, 123 timeout_sec=30) 124 125 return tab 126 127 def tear_down_webpage(self): 128 """Close the tab containing testing webpage.""" 129 # Do not close the tab when the sampler_callback is 130 # doing its work. 131 with self.sampler_lock: 132 self.active_tab.Close() 133 self.active_tab = None 134 135 def run_fish_test(self, browser, test_url, num_fishes, perf_log=True): 136 """Run the test with the given number of fishes. 137 138 @param browser: The Browser object to run the test with. 139 @param test_url: The URL to the aquarium test site. 140 @param num_fishes: The number of fishes to run the test with. 141 @param perf_log: Report perf data only if it's set to True. 142 """ 143 144 tab = self.setup_webpage(browser, test_url, num_fishes) 145 146 if self.kernel_sampler: 147 self.kernel_sampler.start_sampling_thread() 148 time.sleep(self.test_duration_secs) 149 if self.kernel_sampler: 150 self.kernel_sampler.stop_sampling_thread() 151 self.kernel_sampler.output_flip_stats('flip_stats_%d' % num_fishes) 152 self.flip_stats = {} 153 154 # Get average FPS and rendering time, then close the tab. 155 avg_fps = tab.EvaluateJavaScript('g_crosFpsCounter.getAvgFps();') 156 if math.isnan(float(avg_fps)): 157 raise error.TestFail('Failed: Could not get FPS count.') 158 159 avg_interframe_time = tab.EvaluateJavaScript( 160 'g_crosFpsCounter.getAvgInterFrameTime();') 161 avg_render_time = tab.EvaluateJavaScript( 162 'g_crosFpsCounter.getAvgRenderTime();') 163 std_interframe_time = tab.EvaluateJavaScript( 164 'g_crosFpsCounter.getStdInterFrameTime();') 165 std_render_time = tab.EvaluateJavaScript( 166 'g_crosFpsCounter.getStdRenderTime();') 167 self.perf_keyval['avg_fps_%04d_fishes' % num_fishes] = avg_fps 168 self.perf_keyval['avg_interframe_time_%04d_fishes' % num_fishes] = ( 169 avg_interframe_time) 170 self.perf_keyval['avg_render_time_%04d_fishes' % num_fishes] = ( 171 avg_render_time) 172 self.perf_keyval['std_interframe_time_%04d_fishes' % num_fishes] = ( 173 std_interframe_time) 174 self.perf_keyval['std_render_time_%04d_fishes' % num_fishes] = ( 175 std_render_time) 176 logging.info('%d fish(es): Average FPS = %f, ' 177 'average render time = %f', num_fishes, avg_fps, 178 avg_render_time) 179 180 if perf_log: 181 # Report frames per second to chromeperf/ dashboard. 182 self.output_perf_value( 183 description='avg_fps_%04d_fishes' % num_fishes, 184 value=avg_fps, 185 units='fps', 186 higher_is_better=True) 187 188 # Intel only: Record the power consumption for the next few seconds. 189 rapl_rate = power_rapl.get_rapl_measurement( 190 'rapl_%04d_fishes' % num_fishes) 191 # Remove entries that we don't care about. 192 rapl_rate = {key: rapl_rate[key] 193 for key in rapl_rate.keys() if key.endswith('pwr')} 194 # Report to chromeperf/ dashboard. 195 for key, values in rapl_rate.iteritems(): 196 self.output_perf_value( 197 description=key, 198 value=values, 199 units='W', 200 higher_is_better=False, 201 graph='rapl_power_consumption' 202 ) 203 204 def run_power_test(self, browser, test_url, ac_ok): 205 """Runs the webgl power consumption test and reports the perf results. 206 207 @param browser: The Browser object to run the test with. 208 @param test_url: The URL to the aquarium test site. 209 @param ac_ok: Boolean on whether its ok to have AC power supplied. 210 """ 211 212 self._backlight = power_utils.Backlight() 213 self._backlight.set_default() 214 215 self._service_stopper = service_stopper.ServiceStopper( 216 service_stopper.ServiceStopper.POWER_DRAW_SERVICES) 217 self._service_stopper.stop_services() 218 219 if not ac_ok: 220 self._power_status = power_status.get_status() 221 # Verify that we are running on battery and the battery is 222 # sufficiently charged. 223 self._power_status.assert_battery_state(BATTERY_INITIAL_CHARGED_MIN) 224 225 measurements = [ 226 power_status.SystemPower(self._power_status.battery_path) 227 ] 228 229 def get_power(): 230 power_logger = power_status.PowerLogger(measurements) 231 power_logger.start() 232 time.sleep(STABILIZATION_DURATION) 233 start_time = time.time() 234 time.sleep(MEASUREMENT_DURATION) 235 power_logger.checkpoint('result', start_time) 236 keyval = power_logger.calc() 237 logging.info('Power output %s', keyval) 238 return keyval['result_' + measurements[0].domain + '_pwr'] 239 240 self.run_fish_test(browser, test_url, 1000, perf_log=False) 241 if not ac_ok: 242 energy_rate = get_power() 243 # This is a power specific test so we are not capturing 244 # avg_fps and avg_render_time in this test. 245 self.perf_keyval[POWER_DESCRIPTION] = energy_rate 246 self.output_perf_value( 247 description=POWER_DESCRIPTION, 248 value=energy_rate, 249 units='W', 250 higher_is_better=False) 251 252 def exynos_sampler_callback(self, sampler_obj): 253 """Sampler callback function for ExynosSampler. 254 255 @param sampler_obj: The ExynosSampler object that invokes this callback 256 function. 257 """ 258 if sampler_obj.stopped: 259 return 260 261 with self.sampler_lock: 262 now = time.time() 263 results = {} 264 info_str = ['\nfb_id wait_kds flipped'] 265 for value in sampler_obj.frame_buffers.itervalues(): 266 results[value.fb] = {} 267 for state, stats in value.states.iteritems(): 268 results[value.fb][state] = (stats.avg, stats.stdev) 269 info_str.append('%s: %s %s' % (value.fb, 270 results[value.fb]['wait_kds'][0], 271 results[value.fb]['flipped'][0])) 272 results['avg_fps'] = self.active_tab.EvaluateJavaScript( 273 'g_crosFpsCounter.getAvgFps();') 274 results['avg_render_time'] = self.active_tab.EvaluateJavaScript( 275 'g_crosFpsCounter.getAvgRenderTime();') 276 self.active_tab.ExecuteJavaScript('g_crosFpsCounter.reset();') 277 info_str.append('avg_fps: %s, avg_render_time: %s' % 278 (results['avg_fps'], results['avg_render_time'])) 279 self.flip_stats[now] = results 280 logging.info('\n'.join(info_str)) 281 282 def exynos_output_flip_stats(self, file_name): 283 """Pageflip statistics output function for ExynosSampler. 284 285 @param file_name: The output file name. 286 """ 287 # output format: 288 # time fb_id avg_rendered avg_prepared avg_wait_kds avg_flipped 289 # std_rendered std_prepared std_wait_kds std_flipped 290 with open(file_name, 'w') as f: 291 for t in sorted(self.flip_stats.keys()): 292 if ('avg_fps' in self.flip_stats[t] and 293 'avg_render_time' in self.flip_stats[t]): 294 f.write('%s %s %s\n' % 295 (t, self.flip_stats[t]['avg_fps'], 296 self.flip_stats[t]['avg_render_time'])) 297 for fb, stats in self.flip_stats[t].iteritems(): 298 if not isinstance(fb, int): 299 continue 300 f.write('%s %s ' % (t, fb)) 301 f.write('%s %s %s %s ' % (stats['rendered'][0], 302 stats['prepared'][0], 303 stats['wait_kds'][0], 304 stats['flipped'][0])) 305 f.write('%s %s %s %s\n' % (stats['rendered'][1], 306 stats['prepared'][1], 307 stats['wait_kds'][1], 308 stats['flipped'][1])) 309 310 def write_samples(self, samples, filename): 311 """Writes all samples to result dir with the file name "samples'. 312 313 @param samples: A list of all collected samples. 314 @param filename: The file name to save under result directory. 315 """ 316 out_file = os.path.join(self.resultsdir, filename) 317 with open(out_file, 'w') as f: 318 for sample in samples: 319 print >> f, sample 320 321 def run_fish_test_with_memory_pressure( 322 self, browser, test_url, num_fishes, memory_pressure): 323 """Measure fps under memory pressure. 324 325 It measure FPS of WebGL aquarium while adding memory pressure. It runs 326 in 2 phases: 327 1. Allocate non-swappable memory until |memory_to_reserve_mb| is 328 remained. The memory is not accessed after allocated. 329 2. Run "active" memory consumer in the background. After allocated, 330 Its content is accessed sequentially by page and looped around 331 infinitely. 332 The second phase is opeared in two possible modes: 333 1. "single" mode, which means only one "active" memory consumer. After 334 running a single memory consumer with a given memory size, it waits 335 for a while to see if system can afford current memory pressure 336 (definition here is FPS > 5). If it does, kill current consumer and 337 launch another consumer with a larger memory size. The process keeps 338 going until system couldn't afford the load. 339 2. "multiple"mode. It simply launch memory consumers with a given size 340 one by one until system couldn't afford the load (e.g., FPS < 5). 341 In "single" mode, CPU load is lighter so we expect swap in/swap out 342 rate to be correlated to FPS better. In "multiple" mode, since there 343 are multiple busy loop processes, CPU pressure is another significant 344 cause of frame drop. Frame drop can happen easily due to busy CPU 345 instead of memory pressure. 346 347 @param browser: The Browser object to run the test with. 348 @param test_url: The URL to the aquarium test site. 349 @param num_fishes: The number of fishes to run the test with. 350 @param memory_pressure: Memory pressure parameters. 351 """ 352 consumer_mode = memory_pressure.get('consumer_mode', 'single') 353 memory_to_reserve_mb = memory_pressure.get('memory_to_reserve_mb', 500) 354 # Empirical number to quickly produce memory pressure. 355 if consumer_mode == 'single': 356 default_consumer_size_mb = memory_to_reserve_mb + 100 357 else: 358 default_consumer_size_mb = memory_to_reserve_mb / 2 359 consumer_size_mb = memory_pressure.get( 360 'consumer_size_mb', default_consumer_size_mb) 361 362 # Setup fish tank. 363 self.setup_webpage(browser, test_url, num_fishes) 364 365 # Drop all file caches. 366 utils.drop_caches() 367 368 def fps_near_zero(fps_sampler): 369 """Returns whether recent fps goes down to near 0. 370 371 @param fps_sampler: A system_sampler.Sampler object. 372 """ 373 last_fps = fps_sampler.get_last_avg_fps(6) 374 if last_fps: 375 logging.info('last fps %f', last_fps) 376 if last_fps <= 5: 377 return True 378 return False 379 380 max_allocated_mb = 0 381 # Consume free memory and release them by the end. 382 with memory_eater.consume_free_memory(memory_to_reserve_mb): 383 fps_sampler = system_sampler.SystemSampler( 384 memory_eater.MemoryEater.get_active_consumer_pids) 385 end_condition = functools.partial(fps_near_zero, fps_sampler) 386 with fps_meter.FPSMeter(fps_sampler.sample): 387 # Collects some samples before running memory pressure. 388 time.sleep(5) 389 try: 390 if consumer_mode == 'single': 391 # A single run couldn't generate samples representative 392 # enough. 393 # First runs squeeze more inactive anonymous memory into 394 # zram so in later runs we have a more stable memory 395 # stat. 396 max_allocated_mb = max( 397 memory_eater.run_single_memory_pressure( 398 consumer_size_mb, 100, end_condition, 10, 3, 399 900), 400 memory_eater.run_single_memory_pressure( 401 consumer_size_mb, 20, end_condition, 10, 3, 402 900), 403 memory_eater.run_single_memory_pressure( 404 consumer_size_mb, 10, end_condition, 10, 3, 405 900)) 406 elif consumer_mode == 'multiple': 407 max_allocated_mb = ( 408 memory_eater.run_multi_memory_pressure( 409 consumer_size_mb, end_condition, 10, 900)) 410 else: 411 raise error.TestFail( 412 'Failed: Unsupported consumer mode.') 413 except memory_eater.TimeoutException as e: 414 raise error.TestFail(e) 415 416 samples = fps_sampler.get_samples() 417 self.write_samples(samples, 'memory_pressure_fps_samples.txt') 418 419 self.perf_keyval['num_samples'] = len(samples) 420 self.perf_keyval['max_allocated_mb'] = max_allocated_mb 421 422 logging.info(self.perf_keyval) 423 424 self.output_perf_value( 425 description='max_allocated_mb_%d_fishes_reserved_%d_mb' % ( 426 num_fishes, memory_to_reserve_mb), 427 value=max_allocated_mb, 428 units='MB', 429 higher_is_better=True) 430 431 432 @graphics_utils.GraphicsTest.failure_report_decorator('graphics_WebGLAquarium') 433 def run_once(self, 434 test_duration_secs=30, 435 test_setting_num_fishes=(50, 1000), 436 power_test=False, 437 ac_ok=False, 438 memory_pressure=None): 439 """Find a browser with telemetry, and run the test. 440 441 @param test_duration_secs: The duration in seconds to run each scenario 442 for. 443 @param test_setting_num_fishes: A list of the numbers of fishes to 444 enable in the test. 445 @param power_test: Boolean on whether to run power_test 446 @param ac_ok: Boolean on whether its ok to have AC power supplied. 447 @param memory_pressure: A dictionay which specifies memory pressure 448 parameters: 449 'consumer_mode': 'single' or 'multiple' to have one or moultiple 450 concurrent memory consumers. 451 'consumer_size_mb': Amount of memory to allocate. In 'single' 452 mode, a single memory consumer would allocate memory by the 453 specific size. It then gradually allocates more memory until 454 FPS down to near 0. In 'multiple' mode, memory consumers of 455 this size would be spawn one by one until FPS down to near 0. 456 'memory_to_reserve_mb': Amount of memory to reserve before 457 running memory consumer. In practical we allocate mlocked 458 memory (i.e., not swappable) to consume free memory until this 459 amount of free memory remained. 460 """ 461 self.test_duration_secs = test_duration_secs 462 self.test_setting_num_fishes = test_setting_num_fishes 463 464 with chrome.Chrome(logged_in=False, init_network_controller=True) as cr: 465 cr.browser.platform.SetHTTPServerDirectories(self.srcdir) 466 test_url = cr.browser.platform.http_server.UrlOf( 467 os.path.join(self.srcdir, 'aquarium.html')) 468 469 if not utils.wait_for_idle_cpu(60.0, 0.1): 470 if not utils.wait_for_idle_cpu(20.0, 0.2): 471 raise error.TestFail('Failed: Could not get idle CPU.') 472 if not utils.wait_for_cool_machine(): 473 raise error.TestFail('Failed: Could not get cold machine.') 474 if memory_pressure: 475 self.run_fish_test_with_memory_pressure( 476 cr.browser, test_url, num_fishes=1000, 477 memory_pressure=memory_pressure) 478 self.tear_down_webpage() 479 elif power_test: 480 self._test_power = True 481 self.run_power_test(cr.browser, test_url, ac_ok) 482 self.tear_down_webpage() 483 else: 484 for n in self.test_setting_num_fishes: 485 self.run_fish_test(cr.browser, test_url, n) 486 self.tear_down_webpage() 487 self.write_perf_keyval(self.perf_keyval) 488
