1 # 2 # Copyright (C) 2016 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # 16 17 import logging 18 import os 19 import xml.etree.ElementTree 20 21 from vts.runners.host import asserts 22 from vts.runners.host import const 23 from vts.runners.host import keys 24 from vts.runners.host import test_runner 25 26 from vts.testcases.template.binary_test import binary_test 27 from vts.testcases.template.binary_test import binary_test_case 28 from vts.testcases.template.gtest_binary_test import gtest_test_case 29 30 31 class GtestBinaryTest(binary_test.BinaryTest): 32 '''Base class to run gtests binary on target. 33 34 Attributes: 35 DEVICE_TEST_DIR: string, temp location for storing binary 36 TAG_PATH_SEPARATOR: string, separator used to separate tag and path 37 shell: ShellMirrorObject, shell mirror 38 tags: all the tags that appeared in binary list 39 testcases: list of GtestTestCase objects, list of test cases to run 40 _dut: AndroidDevice, the device under test as config 41 _gtest_results: list of GtestResult objects, used during batch mode 42 for result storage and parsing 43 ''' 44 45 # @Override 46 def setUpClass(self): 47 '''Prepare class, push binaries, set permission, create test cases.''' 48 self.batch_mode = self.getUserParam( 49 keys.ConfigKeys.IKEY_GTEST_BATCH_MODE, default_value=False) 50 if self.batch_mode: 51 self._gtest_results = [] 52 super(GtestBinaryTest, self).setUpClass() 53 54 # @Override 55 def CreateTestCase(self, path, tag=''): 56 '''Create a list of GtestTestCase objects from a binary path. 57 58 Args: 59 path: string, absolute path of a gtest binary on device 60 tag: string, a tag that will be appended to the end of test name 61 62 Returns: 63 A list of GtestTestCase objects on success; an empty list otherwise. 64 In non-batch mode, each object respresents a test case in the 65 gtest binary located at the provided path. Usually there are more 66 than one object returned. 67 In batch mode, each object represents a gtest binary located at 68 the provided path; the returned list will always be a one object 69 list in batch mode. Test case names are stored in full_name 70 property in the object, delimited by ':' according to gtest 71 documentation, after being filtered and processed according to 72 host configuration. 73 ''' 74 working_directory = self.working_directory[ 75 tag] if tag in self.working_directory else None 76 envp = self.envp[tag] if tag in self.envp else '' 77 args = self.args[tag] if tag in self.args else '' 78 ld_library_path = self.ld_library_path[ 79 tag] if tag in self.ld_library_path else None 80 profiling_library_path = self.profiling_library_path[ 81 tag] if tag in self.profiling_library_path else None 82 83 gtest_list_args = args + " --gtest_list_tests" 84 list_test_case = binary_test_case.BinaryTestCase( 85 'gtest_list_tests', 86 path, 87 path, 88 tag, 89 self.PutTag, 90 working_directory, 91 ld_library_path, 92 profiling_library_path, 93 envp=envp, 94 args=gtest_list_args) 95 cmd = ['chmod 755 %s' % path, list_test_case.GetRunCommand()] 96 cmd_results = self.shell.Execute(cmd) 97 test_cases = [] 98 if any(cmd_results[const.EXIT_CODE] 99 ): # gtest binary doesn't exist or is corrupted 100 logging.error( 101 'Failed to list test cases from %s. Command: %s, Result: %s.' % 102 (path, cmd, cmd_results)) 103 return test_cases 104 105 test_suite = '' 106 for line in cmd_results[const.STDOUT][1].split('\n'): 107 line = str(line) 108 if not len(line.strip()): 109 continue 110 elif line.startswith(' '): # Test case name 111 test_name = line.split('#')[0].strip() 112 test_case = gtest_test_case.GtestTestCase( 113 test_suite, test_name, path, tag, self.PutTag, 114 working_directory, ld_library_path, profiling_library_path, 115 envp=envp, args=args) 116 logging.info('Gtest test case: %s' % test_case) 117 test_cases.append(test_case) 118 else: # Test suite name 119 test_suite = line.strip() 120 if test_suite.endswith('.'): 121 test_suite = test_suite[:-1] 122 123 if not self.batch_mode: 124 return test_cases 125 126 # Gtest batch mode 127 test_names = map(lambda test: test.full_name, test_cases) 128 129 gtest_batch = gtest_test_case.GtestTestCase( 130 path, '', path, tag, self.PutTag, working_directory, 131 ld_library_path, profiling_library_path) 132 gtest_batch.full_name = ':'.join(test_names) 133 return [gtest_batch] 134 135 # @Override 136 def VerifyTestResult(self, test_case, command_results): 137 '''Parse Gtest xml result output. 138 139 Sample 140 <testsuites tests="1" failures="1" disabled="0" errors="0" 141 timestamp="2017-05-24T18:32:10" time="0.012" name="AllTests"> 142 <testsuite name="ConsumerIrHidlTest" 143 tests="1" failures="1" disabled="0" errors="0" time="0.01"> 144 <testcase name="TransmitTest" status="run" time="0.01" 145 classname="ConsumerIrHidlTest"> 146 <failure message="hardware/interfaces..." type=""> 147 <![CDATA[hardware/interfaces...]]> 148 </failure> 149 </testcase> 150 </testsuite> 151 </testsuites> 152 153 Args: 154 test_case: GtestTestCase object, the test being run. This param 155 is not currently used in this method. 156 command_results: dict of lists, shell command result 157 ''' 158 asserts.assertTrue(command_results, 'Empty command response.') 159 asserts.assertEqual( 160 len(command_results), 3, 'Abnormal command response.') 161 for item in command_results.values(): 162 asserts.assertEqual( 163 len(item), 2, 164 'Abnormal command result length: %s' % command_results) 165 166 for stderr in command_results[const.STDERR]: 167 if stderr and stderr.strip(): 168 for line in stderr.split('\n'): 169 logging.error(line) 170 171 xml_str = command_results[const.STDOUT][1].strip() 172 173 if self.batch_mode: 174 self._ParseBatchResults(test_case, xml_str) 175 return 176 177 for stdout in command_results[const.STDOUT]: 178 if stdout and stdout.strip(): 179 for line in stdout.split('\n'): 180 logging.info(line) 181 182 asserts.assertFalse( 183 command_results[const.EXIT_CODE][1], 184 'Failed to show Gtest XML output: %s' % command_results) 185 186 root = xml.etree.ElementTree.fromstring(xml_str) 187 asserts.assertEqual(root.get('tests'), '1', 'No tests available') 188 if root.get('errors') != '0' or root.get('failures') != '0': 189 messages = [x.get('message') for x in root.findall('.//failure')] 190 asserts.fail('\n'.join([x for x in messages if x])) 191 asserts.skipIf(root.get('disabled') == '1', 'Gtest test case disabled') 192 193 def _ParseBatchResults(self, test_case_original, xml_str): 194 '''Parse batch mode gtest results 195 196 Args: 197 test_case_original: GtestTestCase object, original batch test case object 198 xml_str: string, result xml output content 199 ''' 200 root = xml.etree.ElementTree.fromstring(xml_str) 201 202 for test_suite in root: 203 print test_suite.tag, test_suite.attrib 204 for test_case in test_suite: 205 result = gtest_test_case.GtestTestCase( 206 test_suite.get('name'), 207 test_case.get('name'), '', test_case_original.tag, 208 self.PutTag) 209 210 failure_message = None 211 for sub in test_case: 212 if sub.tag == 'failure': 213 failure_message = sub.get('message') 214 215 if len(test_case) and not failure_message: 216 failure_message = 'Error: %s\n' % test_case.attrib 217 for sub in test_case: 218 failure_message += '%s: %s\n' % (sub.tag, sub.attrib) 219 220 result.failure_message = failure_message 221 222 self._gtest_results.append(result) 223 224 def _VerifyBatchResult(self, gtest_result): 225 '''Check a gtest test case result in batch mode 226 227 Args: 228 gtest_result: GtestTestCase object, representing gtest result 229 ''' 230 asserts.assertFalse(gtest_result.failure_message, 231 gtest_result.failure_message) 232 233 # @Override 234 def generateAllTests(self): 235 '''Runs all binary tests.''' 236 if self.batch_mode: 237 for test_case in self.testcases: 238 self.RunTestCase(test_case) 239 240 self.runGeneratedTests( 241 test_func=self._VerifyBatchResult, 242 settings=self._gtest_results, 243 name_func=str) 244 245 self._gtest_results = [] 246 return 247 248 self.runGeneratedTests( 249 test_func=self.RunTestCase, settings=self.testcases, name_func=str) 250 251 252 if __name__ == "__main__": 253 test_runner.main() 254
