1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 # 5 # GLMark outputs a final performance score, and it checks the performance score 6 # against minimum requirement if min_score is set. 7 8 import logging 9 import os 10 import re 11 import string 12 13 from autotest_lib.client.bin import test, utils 14 from autotest_lib.client.common_lib import error 15 from autotest_lib.client.cros import service_stopper 16 from autotest_lib.client.cros.graphics import graphics_utils 17 18 GLMARK2_TEST_RE = ( 19 r'^\[(?P<scene>.*)\] (?P<options>.*): FPS: (?P<fps>\d+) FrameTime: ' 20 r'(?P<frametime>\d+.\d+) ms$') 21 GLMARK2_SCORE_RE = r'glmark2 Score: (\d+)' 22 23 # perf value description strings may only contain letters, numbers, periods, 24 # dashes and underscores. 25 # But glmark2 test names are usually in the form: 26 # scene-name:opt=val:opt=v1,v2;v3,v4 or scene:<default> 27 # which we convert to: 28 # scene-name.opt_val.opt_v1-v2_v3-v4 or scene.default 29 description_table = string.maketrans(':,=;', '.-__') 30 description_delete = '<>' 31 32 33 class graphics_GLMark2(test.test): 34 """Runs glmark2, which benchmarks only calls compatible with OpenGL ES 2.0""" 35 version = 1 36 preserve_srcdir = True 37 _services = None 38 GSC = None 39 40 def setup(self): 41 self.job.setup_dep(['glmark2']) 42 43 def initialize(self): 44 self.GSC = graphics_utils.GraphicsStateChecker() 45 # If UI is running, we must stop it and restore later. 46 self._services = service_stopper.ServiceStopper(['ui']) 47 self._services.stop_services() 48 49 def cleanup(self): 50 if self._services: 51 self._services.restore_services() 52 if self.GSC: 53 keyvals = self.GSC.get_memory_keyvals() 54 for key, val in keyvals.iteritems(): 55 self.output_perf_value( 56 description=key, 57 value=val, 58 units='bytes', 59 higher_is_better=False) 60 self.GSC.finalize() 61 self.write_perf_keyval(keyvals) 62 63 def run_once(self, size='800x600', hasty=False, min_score=None): 64 dep = 'glmark2' 65 dep_dir = os.path.join(self.autodir, 'deps', dep) 66 self.job.install_pkg(dep, 'dep', dep_dir) 67 68 glmark2 = os.path.join(self.autodir, 'deps/glmark2/glmark2') 69 if not os.path.exists(glmark2): 70 raise error.TestFail('Failed: Could not find test binary.') 71 72 glmark2_data = os.path.join(self.autodir, 'deps/glmark2/data') 73 74 options = [] 75 options.append('--data-path %s' % glmark2_data) 76 options.append('--size %s' % size) 77 options.append('--annotate') 78 if hasty: 79 options.append('-b :duration=0.2') 80 else: 81 options.append('-b :duration=2') 82 cmd = glmark2 + ' ' + ' '.join(options) 83 84 if os.environ.get('CROS_FACTORY'): 85 from autotest_lib.client.cros import factory_setup_modules 86 from cros.factory.test import ui 87 ui.start_reposition_thread('^glmark') 88 89 # TODO(ihf): Switch this test to use perf.PerfControl like 90 # graphics_GLBench once it is stable. crbug.com/344766. 91 if not hasty: 92 if not utils.wait_for_idle_cpu(60.0, 0.1): 93 if not utils.wait_for_idle_cpu(20.0, 0.2): 94 raise error.TestFail('Failed: Could not get idle CPU.') 95 if not utils.wait_for_cool_machine(): 96 raise error.TestFail('Failed: Could not get cold machine.') 97 98 # In this test we are manually handling stderr, so expected=True. 99 # Strangely autotest takes CmdError/CmdTimeoutError as warning only. 100 try: 101 result = utils.run(cmd, 102 stderr_is_expected=True, 103 stdout_tee=utils.TEE_TO_LOGS, 104 stderr_tee=utils.TEE_TO_LOGS) 105 except error.CmdError: 106 raise error.TestFail('Failed: CmdError running %s' % cmd) 107 except error.CmdTimeoutError: 108 raise error.TestFail('Failed: CmdTimeout running %s' % cmd) 109 110 logging.info(result) 111 for line in result.stderr.splitlines(): 112 if line.startswith('Error:'): 113 # Line already starts with 'Error: ", not need to prepend. 114 raise error.TestFail(line) 115 116 # Numbers in hasty mode are not as reliable, so don't send them to 117 # the dashboard etc. 118 if not hasty: 119 keyvals = {} 120 score = None 121 test_re = re.compile(GLMARK2_TEST_RE) 122 for line in result.stdout.splitlines(): 123 match = test_re.match(line) 124 if match: 125 test = '%s.%s' % (match.group('scene'), 126 match.group('options')) 127 test = test.translate(description_table, description_delete) 128 frame_time = match.group('frametime') 129 keyvals[test] = frame_time 130 self.output_perf_value( 131 description=test, 132 value=frame_time, 133 units='ms', 134 higher_is_better=False) 135 else: 136 # glmark2 output the final performance score as: 137 # glmark2 Score: 530 138 match = re.findall(GLMARK2_SCORE_RE, line) 139 if match: 140 score = int(match[0]) 141 if score is None: 142 raise error.TestFail('Failed: Unable to read benchmark score') 143 # Output numbers for plotting by harness. 144 logging.info('GLMark2 score: %d', score) 145 if os.environ.get('CROS_FACTORY'): 146 from autotest_lib.client.cros import factory_setup_modules 147 from cros.factory.event_log import EventLog 148 EventLog('graphics_GLMark2').Log('glmark2_score', score=score) 149 keyvals['glmark2_score'] = score 150 self.write_perf_keyval(keyvals) 151 self.output_perf_value( 152 description='Score', 153 value=score, 154 units='score', 155 higher_is_better=True) 156 157 if min_score is not None and score < min_score: 158 raise error.TestFail( 159 'Failed: Benchmark score %d < %d (minimum score ' 160 'requirement)' % (score, min_score)) 161
