1 # 2 # Copyright (C) 2016 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # 16 17 import logging 18 import os 19 import xml.etree.ElementTree 20 21 from vts.runners.host import asserts 22 from vts.runners.host import const 23 from vts.runners.host import keys 24 from vts.runners.host import test_runner 25 26 from vts.testcases.template.binary_test import binary_test 27 from vts.testcases.template.binary_test import binary_test_case 28 from vts.testcases.template.gtest_binary_test import gtest_test_case 29 30 31 class GtestBinaryTest(binary_test.BinaryTest): 32 '''Base class to run gtests binary on target. 33 34 Attributes: 35 DEVICE_TEST_DIR: string, temp location for storing binary 36 TAG_PATH_SEPARATOR: string, separator used to separate tag and path 37 shell: ShellMirrorObject, shell mirror 38 tags: all the tags that appeared in binary list 39 testcases: list of GtestTestCase objects, list of test cases to run 40 _dut: AndroidDevice, the device under test as config 41 _gtest_results: list of GtestResult objects, used during batch mode 42 for result storage and parsing 43 ''' 44 45 # @Override 46 def setUpClass(self): 47 '''Prepare class, push binaries, set permission, create test cases.''' 48 self.collect_tests_only = self.getUserParam( 49 keys.ConfigKeys.IKEY_COLLECT_TESTS_ONLY, default_value=False) 50 self.batch_mode = self.getUserParam( 51 keys.ConfigKeys.IKEY_GTEST_BATCH_MODE, default_value=False) 52 53 if self.batch_mode: 54 if self.collect_tests_only: 55 self.batch_mode = False 56 logging.info("Disable batch mode when collecting tests.") 57 else: 58 self._gtest_results = [] 59 60 super(GtestBinaryTest, self).setUpClass() 61 62 # @Override 63 def CreateTestCase(self, path, tag=''): 64 '''Create a list of GtestTestCase objects from a binary path. 65 66 Args: 67 path: string, absolute path of a gtest binary on device 68 tag: string, a tag that will be appended to the end of test name 69 70 Returns: 71 A list of GtestTestCase objects on success; an empty list otherwise. 72 In non-batch mode, each object respresents a test case in the 73 gtest binary located at the provided path. Usually there are more 74 than one object returned. 75 In batch mode, each object represents a gtest binary located at 76 the provided path; the returned list will always be a one object 77 list in batch mode. Test case names are stored in full_name 78 property in the object, delimited by ':' according to gtest 79 documentation, after being filtered and processed according to 80 host configuration. 81 ''' 82 working_directory = self.working_directory[ 83 tag] if tag in self.working_directory else None 84 envp = self.envp[tag] if tag in self.envp else '' 85 args = self.args[tag] if tag in self.args else '' 86 ld_library_path = self.ld_library_path[ 87 tag] if tag in self.ld_library_path else None 88 profiling_library_path = self.profiling_library_path[ 89 tag] if tag in self.profiling_library_path else None 90 91 gtest_list_args = args + " --gtest_list_tests" 92 list_test_case = binary_test_case.BinaryTestCase( 93 'gtest_list_tests', 94 path, 95 path, 96 tag, 97 self.PutTag, 98 working_directory, 99 ld_library_path, 100 profiling_library_path, 101 envp=envp, 102 args=gtest_list_args) 103 cmd = ['chmod 755 %s' % path, list_test_case.GetRunCommand()] 104 cmd_results = self.shell.Execute(cmd) 105 test_cases = [] 106 if any(cmd_results[const.EXIT_CODE] 107 ): # gtest binary doesn't exist or is corrupted 108 logging.error( 109 'Failed to list test cases from %s. Command: %s, Result: %s.' % 110 (path, cmd, cmd_results)) 111 return test_cases 112 113 test_suite = '' 114 for line in cmd_results[const.STDOUT][1].split('\n'): 115 line = str(line) 116 if not len(line.strip()): 117 continue 118 elif line.startswith(' '): # Test case name 119 test_name = line.split('#')[0].strip() 120 test_case = gtest_test_case.GtestTestCase( 121 test_suite, test_name, path, tag, self.PutTag, 122 working_directory, ld_library_path, profiling_library_path, 123 envp=envp, args=args) 124 logging.info('Gtest test case: %s' % test_case) 125 test_cases.append(test_case) 126 else: # Test suite name 127 test_suite = line.strip() 128 if test_suite.endswith('.'): 129 test_suite = test_suite[:-1] 130 131 if not self.batch_mode: 132 return test_cases 133 134 # Gtest batch mode 135 test_names = map(lambda test: test.full_name, test_cases) 136 137 gtest_batch = gtest_test_case.GtestTestCase( 138 path, '', path, tag, self.PutTag, working_directory, 139 ld_library_path, profiling_library_path, envp=envp) 140 gtest_batch.full_name = ':'.join(test_names) 141 return [gtest_batch] 142 143 # @Override 144 def VerifyTestResult(self, test_case, command_results): 145 '''Parse Gtest xml result output. 146 147 Sample 148 <testsuites tests="1" failures="1" disabled="0" errors="0" 149 timestamp="2017-05-24T18:32:10" time="0.012" name="AllTests"> 150 <testsuite name="ConsumerIrHidlTest" 151 tests="1" failures="1" disabled="0" errors="0" time="0.01"> 152 <testcase name="TransmitTest" status="run" time="0.01" 153 classname="ConsumerIrHidlTest"> 154 <failure message="hardware/interfaces..." type=""> 155 <![CDATA[hardware/interfaces...]]> 156 </failure> 157 </testcase> 158 </testsuite> 159 </testsuites> 160 161 Args: 162 test_case: GtestTestCase object, the test being run. This param 163 is not currently used in this method. 164 command_results: dict of lists, shell command result 165 ''' 166 asserts.assertTrue(command_results, 'Empty command response.') 167 asserts.assertEqual( 168 len(command_results), 3, 'Abnormal command response.') 169 for item in command_results.values(): 170 asserts.assertEqual( 171 len(item), 2, 172 'Abnormal command result length: %s' % command_results) 173 174 for stderr in command_results[const.STDERR]: 175 if stderr and stderr.strip(): 176 for line in stderr.split('\n'): 177 logging.error(line) 178 179 xml_str = command_results[const.STDOUT][1].strip() 180 181 if self.batch_mode: 182 self._ParseBatchResults(test_case, xml_str) 183 return 184 185 for stdout in command_results[const.STDOUT]: 186 if stdout and stdout.strip(): 187 for line in stdout.split('\n'): 188 logging.info(line) 189 190 asserts.assertFalse( 191 command_results[const.EXIT_CODE][1], 192 'Failed to show Gtest XML output: %s' % command_results) 193 194 root = xml.etree.ElementTree.fromstring(xml_str) 195 asserts.assertEqual(root.get('tests'), '1', 'No tests available') 196 if root.get('errors') != '0' or root.get('failures') != '0': 197 messages = [x.get('message') for x in root.findall('.//failure')] 198 asserts.fail('\n'.join([x for x in messages if x])) 199 asserts.skipIf(root.get('disabled') == '1', 'Gtest test case disabled') 200 201 def _ParseBatchResults(self, test_case_original, xml_str): 202 '''Parse batch mode gtest results 203 204 Args: 205 test_case_original: GtestTestCase object, original batch test case object 206 xml_str: string, result xml output content 207 ''' 208 root = xml.etree.ElementTree.fromstring(xml_str) 209 210 for test_suite in root: 211 print test_suite.tag, test_suite.attrib 212 for test_case in test_suite: 213 result = gtest_test_case.GtestTestCase( 214 test_suite.get('name'), 215 test_case.get('name'), '', test_case_original.tag, 216 self.PutTag, name_appendix=test_case_original.name_appendix) 217 218 failure_message = None 219 for sub in test_case: 220 if sub.tag == 'failure': 221 failure_message = sub.get('message') 222 223 if len(test_case) and not failure_message: 224 failure_message = 'Error: %s\n' % test_case.attrib 225 for sub in test_case: 226 failure_message += '%s: %s\n' % (sub.tag, sub.attrib) 227 228 result.failure_message = failure_message 229 230 self._gtest_results.append(result) 231 232 def _VerifyBatchResult(self, gtest_result): 233 '''Check a gtest test case result in batch mode 234 235 Args: 236 gtest_result: GtestTestCase object, representing gtest result 237 ''' 238 asserts.assertFalse(gtest_result.failure_message, 239 gtest_result.failure_message) 240 241 # @Override 242 def generateAllTests(self): 243 '''Runs all binary tests.''' 244 if self.batch_mode: 245 for test_case in self.testcases: 246 self.RunTestCase(test_case) 247 248 self.runGeneratedTests( 249 test_func=self._VerifyBatchResult, 250 settings=self._gtest_results, 251 name_func=str) 252 253 self._gtest_results = [] 254 return 255 256 self.runGeneratedTests( 257 test_func=self.RunTestCase, settings=self.testcases, name_func=str) 258 259 260 if __name__ == "__main__": 261 test_runner.main() 262
