Home | History | Annotate | Download | only in graphics_WebGLPerformance 
      1 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 """This is a client side WebGL performance test.
      5 
      6 http://hg.mozilla.org/users/bjacob_mozilla.com/webgl-perf-tests/raw-file/3729e8afac99/index.html
      7 
      8 From the sources:
      9 Keep in mind that these tests are not realistic workloads. These are not
     10 benchmarks aiming to compare browser or GPU performance. These are only useful
     11 to catch performance regressions in a given browser and system.
     12 """
     13 
     14 import logging
     15 import os
     16 import math
     17 
     18 from autotest_lib.client.bin import utils
     19 from autotest_lib.client.common_lib import error
     20 from autotest_lib.client.common_lib.cros import chrome
     21 from autotest_lib.client.cros.graphics import graphics_utils
     22 
     23 
     24 class graphics_WebGLPerformance(graphics_utils.GraphicsTest):
     25     """WebGL performance graphics test."""
     26     version = 1
     27     _test_duration_secs = 0
     28     perf_keyval = {}
     29     _waived_tests = ['convert-Canvas-to-rgb-float.html',
     30                      'convert-Canvas-to-rgb-float-premultiplied.html']
     31 
     32     def setup(self):
     33         self.job.setup_dep(['webgl_perf'])
     34         self.job.setup_dep(['graphics'])
     35 
     36     def initialize(self):
     37         super(graphics_WebGLPerformance, self).initialize()
     38 
     39     def cleanup(self):
     40         super(graphics_WebGLPerformance, self).cleanup()
     41 
     42     def run_performance_test(self, browser, test_url):
     43         """Runs the performance test from the given url.
     44 
     45         @param browser: The Browser object to run the test with.
     46         @param test_url: The URL to the performance test site.
     47         """
     48         if not utils.wait_for_idle_cpu(60.0, 0.1):
     49             if not utils.wait_for_idle_cpu(20.0, 0.2):
     50                 raise error.TestFail('Failed: Could not get idle CPU.')
     51 
     52         # Kick off test.
     53         tab = browser.tabs.New()
     54         tab.Navigate(test_url)
     55         tab.Activate()
     56         tab.WaitForDocumentReadyStateToBeComplete()
     57 
     58         # Wait for test completion.
     59         tab.WaitForJavaScriptCondition('test_completed == true',
     60                                        timeout=self._test_duration_secs)
     61 
     62         # Get all the result data
     63         results = tab.EvaluateJavaScript('testsRun')
     64         logging.info('results: %s', results)
     65         # Get the geometric mean of individual runtimes.
     66         sumOfLogResults = 0
     67         sumOfPassed = 0
     68         sumOfFailed = 0
     69         sumOfWaived = 0
     70         for result in results:
     71             if result.get('url') in self._waived_tests or result.get('skip'):
     72                 sumOfWaived += 1
     73             elif 'error' in result:
     74                 self.add_failures(result.get('url'))
     75                 sumOfFailed += 1
     76             else:
     77                 sumOfLogResults += math.log(result['testResult'])
     78                 sumOfPassed += 1
     79         time_ms_geom_mean = round(100 * math.exp(
     80             sumOfLogResults / len(results))) / 100
     81 
     82         logging.info('WebGLPerformance: time_ms_geom_mean = %f',
     83                      time_ms_geom_mean)
     84 
     85         # Output numbers for plotting by harness.
     86         keyvals = {}
     87         keyvals['time_ms_geom_mean'] = time_ms_geom_mean
     88         self.write_perf_keyval(keyvals)
     89         self.output_perf_value(
     90             description='time_geom_mean',
     91             value=time_ms_geom_mean,
     92             units='ms',
     93             higher_is_better=False,
     94             graph='time_geom_mean')
     95         # Add extra value to the graph distinguishing different boards.
     96         variant = utils.get_board_with_frequency_and_memory()
     97         desc = 'time_geom_mean-%s' % variant
     98         self.output_perf_value(
     99             description=desc,
    100             value=time_ms_geom_mean,
    101             units='ms',
    102             higher_is_better=False,
    103             graph='time_geom_mean')
    104 
    105         # Get a copy of the test report.
    106         test_report = tab.EvaluateJavaScript('test_report')
    107         results_path = os.path.join(
    108             self.bindir,
    109             '../../results/default/graphics_WebGLPerformance/test_report.html')
    110         f = open(results_path, 'w+')
    111         f.write(test_report)
    112         f.close()
    113 
    114         tab.Close()
    115         return sumOfPassed, sumOfWaived, sumOfFailed
    116 
    117     def run_once(self, test_duration_secs=2700, fullscreen=True):
    118         """Finds a brower with telemetry, and run the test.
    119 
    120         @param test_duration_secs: The test duration in seconds.
    121         @param fullscreen: Whether to run the test in fullscreen.
    122         """
    123         # To avoid 0ms on fast machines like samus the workload was increased.
    124         # Unfortunately that makes running on slow machines impractical without
    125         # deviating from upstream too much.
    126         if utils.get_gpu_family() == 'pinetrail':
    127             # TODO(ihf): return a TestPass(message) once available.
    128             logging.warning('Test is too slow to run regularly.')
    129             return
    130 
    131         self._test_duration_secs = test_duration_secs
    132         ext_paths = []
    133         if fullscreen:
    134             ext_paths.append(
    135                 os.path.join(self.autodir, 'deps', 'graphics',
    136                              'graphics_test_extension'))
    137 
    138         with chrome.Chrome(logged_in=False,
    139                            extension_paths=ext_paths,
    140                            init_network_controller=True) as cr:
    141             websrc_dir = os.path.join(self.autodir, 'deps', 'webgl_perf', 'src')
    142             if not cr.browser.platform.SetHTTPServerDirectories(websrc_dir):
    143                 raise error.TestFail('Failed: Unable to start HTTP server')
    144             test_url = cr.browser.platform.http_server.UrlOf(
    145                 os.path.join(websrc_dir, 'index.html'))
    146 
    147             passed, waived, failed = self.run_performance_test(cr.browser,
    148                                                                test_url)
    149             logging.debug('Number of tests: %d, passed: %d, '
    150                           'waived: %d, failed: %d',
    151                           passed + waived + failed, passed, waived, failed)
    152             if failed > 0:
    153                 raise error.TestFail('Failed: %d tests failed.' % failed)
    154