1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 # 5 # GLMark outputs a final performance score, and it checks the performance score 6 # against minimum requirement if min_score is set. 7 8 import logging 9 import os 10 import re 11 import string 12 13 from autotest_lib.client.bin import test, utils 14 from autotest_lib.client.common_lib import error 15 from autotest_lib.client.cros import service_stopper 16 from autotest_lib.client.cros.graphics import graphics_utils 17 18 GLMARK2_TEST_RE = ( 19 r'^\[(?P<scene>.*)\] (?P<options>.*): FPS: (?P<fps>\d+) FrameTime: ' 20 r'(?P<frametime>\d+.\d+) ms$') 21 GLMARK2_SCORE_RE = r'glmark2 Score: (\d+)' 22 23 # perf value description strings may only contain letters, numbers, periods, 24 # dashes and underscores. 25 # But glmark2 test names are usually in the form: 26 # scene-name:opt=val:opt=v1,v2;v3,v4 or scene:<default> 27 # which we convert to: 28 # scene-name.opt_val.opt_v1-v2_v3-v4 or scene.default 29 description_table = string.maketrans(':,=;', '.-__') 30 description_delete = '<>' 31 32 33 class graphics_GLMark2(test.test): 34 """Runs glmark2, which benchmarks only calls compatible with OpenGL ES 2.0""" 35 version = 1 36 preserve_srcdir = True 37 _services = None 38 GSC = None 39 40 def setup(self): 41 self.job.setup_dep(['glmark2']) 42 43 def initialize(self): 44 self.GSC = graphics_utils.GraphicsStateChecker() 45 # If UI is running, we must stop it and restore later. 46 self._services = service_stopper.ServiceStopper(['ui']) 47 self._services.stop_services() 48 49 def cleanup(self): 50 if self._services: 51 self._services.restore_services() 52 if self.GSC: 53 keyvals = self.GSC.get_memory_keyvals() 54 for key, val in keyvals.iteritems(): 55 self.output_perf_value(description=key, value=val, 56 units='bytes', higher_is_better=False) 57 self.GSC.finalize() 58 self.write_perf_keyval(keyvals) 59 60 def run_once(self, size='800x600', hasty=False, min_score=None): 61 dep = 'glmark2' 62 dep_dir = os.path.join(self.autodir, 'deps', dep) 63 self.job.install_pkg(dep, 'dep', dep_dir) 64 65 glmark2 = os.path.join(self.autodir, 'deps/glmark2/glmark2') 66 if not os.path.exists(glmark2): 67 raise error.TestFail('Could not find test binary. Setup error.') 68 69 glmark2_data = os.path.join(self.autodir, 'deps/glmark2/data') 70 71 options = [] 72 options.append('--data-path %s' % glmark2_data) 73 options.append('--size %s' % size) 74 options.append('--annotate') 75 if hasty: 76 options.append('-b :duration=0.2') 77 else: 78 options.append('-b :duration=2') 79 cmd = glmark2 + ' ' + ' '.join(options) 80 if not utils.is_freon(): 81 cmd = 'X :1 vt1 & sleep 1; chvt 1 && DISPLAY=:1 ' + cmd 82 83 if os.environ.get('CROS_FACTORY'): 84 from autotest_lib.client.cros import factory_setup_modules 85 from cros.factory.test import ui 86 ui.start_reposition_thread('^glmark') 87 88 # TODO(ihf): Switch this test to use perf.PerfControl like 89 # graphics_GLBench once it is stable. crbug.com/344766. 90 if not hasty: 91 if not utils.wait_for_idle_cpu(60.0, 0.1): 92 if not utils.wait_for_idle_cpu(20.0, 0.2): 93 raise error.TestFail('Could not get idle CPU.') 94 if not utils.wait_for_cool_machine(): 95 raise error.TestFail('Could not get cold machine.') 96 97 try: 98 result = utils.run(cmd, 99 stderr_is_expected=False, 100 stdout_tee=utils.TEE_TO_LOGS, 101 stderr_tee=utils.TEE_TO_LOGS) 102 finally: 103 # Just sending SIGTERM to X is not enough; we must wait for it to 104 # really die before we start a new X server (ie start ui). 105 if not utils.is_freon(): 106 utils.ensure_processes_are_dead_by_name('^X$') 107 108 logging.info(result) 109 for line in result.stderr.splitlines(): 110 if line.startswith('Error:'): 111 raise error.TestFail(line) 112 113 # Numbers in hasty mode are not as reliable, so don't send them to 114 # the dashboard etc. 115 if not hasty: 116 keyvals = {} 117 score = None 118 test_re = re.compile(GLMARK2_TEST_RE) 119 for line in result.stdout.splitlines(): 120 match = test_re.match(line) 121 if match: 122 test = '%s.%s' % (match.group('scene'), 123 match.group('options')) 124 test = test.translate(description_table, 125 description_delete) 126 frame_time = match.group('frametime') 127 keyvals[test] = frame_time 128 self.output_perf_value(description=test, value=frame_time, 129 units='ms', higher_is_better=False) 130 else: 131 # glmark2 output the final performance score as: 132 # glmark2 Score: 530 133 match = re.findall(GLMARK2_SCORE_RE, line) 134 if match: 135 score = int(match[0]) 136 if score is None: 137 raise error.TestFail('Unable to read benchmark score') 138 # Output numbers for plotting by harness. 139 logging.info('GLMark2 score: %d', score) 140 if os.environ.get('CROS_FACTORY'): 141 from autotest_lib.client.cros import factory_setup_modules 142 from cros.factory.event_log import EventLog 143 EventLog('graphics_GLMark2').Log('glmark2_score', score=score) 144 keyvals['glmark2_score'] = score 145 self.write_perf_keyval(keyvals) 146 self.output_perf_value(description='Score', value=score, 147 units='score', higher_is_better=True) 148 149 if min_score is not None and score < min_score: 150 raise error.TestFail('Benchmark score %d < %d (minimum score ' 151 'requirement)' % (score, min_score)) 152
