1 # 2 # Copyright (C) 2017 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # 16 17 import importlib 18 import json 19 import logging 20 import os 21 import sys 22 import time 23 import yaml 24 25 from vts.runners.host import asserts 26 from vts.runners.host import base_test 27 from vts.runners.host import config_parser 28 from vts.runners.host import keys 29 from vts.runners.host import records 30 from vts.runners.host import test_runner 31 from vts.utils.python.io import capture_printout 32 from vts.utils.python.io import file_util 33 34 from mobly import test_runner as mobly_test_runner 35 36 37 LIST_TEST_OUTPUT_START = '==========> ' 38 LIST_TEST_OUTPUT_END = ' <==========' 39 # Temp directory inside python log path. The name is required to be 40 # the set value for tradefed to skip reading contents as logs. 41 TEMP_DIR_NAME = 'temp' 42 CONFIG_FILE_NAME = 'test_config.yaml' 43 MOBLY_RESULT_JSON_FILE_NAME = 'test_run_summary.json' 44 MOBLY_RESULT_YAML_FILE_NAME = 'test_summary.yaml' 45 46 47 MOBLY_CONFIG_TEXT = '''TestBeds: 48 - Name: {module_name} 49 Controllers: 50 AndroidDevice: 51 - serial: {serial1} 52 - serial: {serial2} 53 54 MoblyParams: 55 LogPath: {log_path} 56 ''' 57 58 #TODO(yuexima): 59 # 1. make DEVICES_REQUIRED configurable 60 # 2. add include filter function 61 DEVICES_REQUIRED = 2 62 63 RESULT_KEY_TYPE = 'Type' 64 RESULT_TYPE_SUMMARY = 'Summary' 65 RESULT_TYPE_RECORD = 'Record' 66 RESULT_TYPE_TEST_NAME_LIST = 'TestNameList' 67 RESULT_TYPE_CONTROLLER_INFO = 'ControllerInfo' 68 69 70 class MoblyTest(base_test.BaseTestClass): 71 '''Template class for running mobly test cases. 72 73 Attributes: 74 mobly_dir: string, mobly test temp directory for mobly runner 75 mobly_config_file_path: string, mobly test config file path 76 result_handlers: dict, map of result type and handler functions 77 ''' 78 def setUpClass(self): 79 asserts.assertEqual( 80 len(self.android_devices), DEVICES_REQUIRED, 81 'Exactly %s devices are required for this test.' % DEVICES_REQUIRED 82 ) 83 84 for ad in self.android_devices: 85 logging.info('Android device serial: %s' % ad.serial) 86 87 logging.info('Test cases: %s' % self.ListTestCases()) 88 89 self.mobly_dir = os.path.join(logging.log_path, TEMP_DIR_NAME, 90 'mobly', str(time.time())) 91 92 file_util.Makedirs(self.mobly_dir) 93 94 logging.info('mobly log path: %s' % self.mobly_dir) 95 96 self.result_handlers = { 97 RESULT_TYPE_SUMMARY: self.HandleSimplePrint, 98 RESULT_TYPE_RECORD: self.HandleRecord, 99 RESULT_TYPE_TEST_NAME_LIST: self.HandleSimplePrint, 100 RESULT_TYPE_CONTROLLER_INFO: self.HandleSimplePrint, 101 } 102 103 def tearDownClass(self): 104 ''' Clear the mobly directory.''' 105 file_util.Rmdirs(self.mobly_dir, ignore_errors=True) 106 107 def PrepareConfigFile(self): 108 '''Prepare mobly config file for running test.''' 109 self.mobly_config_file_path = os.path.join(self.mobly_dir, 110 CONFIG_FILE_NAME) 111 config_text = MOBLY_CONFIG_TEXT.format( 112 module_name=self.test_module_name, 113 serial1=self.android_devices[0].serial, 114 serial2=self.android_devices[1].serial, 115 log_path=self.mobly_dir 116 ) 117 with open(self.mobly_config_file_path, 'w') as f: 118 f.write(config_text) 119 120 def ListTestCases(self): 121 '''List test cases. 122 123 Returns: 124 List of string, test names. 125 ''' 126 classes = mobly_test_runner._find_test_class() 127 128 with capture_printout.CaptureStdout() as output: 129 mobly_test_runner._print_test_names(classes) 130 131 test_names = [] 132 133 for line in output: 134 if (not line.startswith(LIST_TEST_OUTPUT_START) 135 and line.endswith(LIST_TEST_OUTPUT_END)): 136 test_names.append(line) 137 tr_record = records.TestResultRecord(line, self.test_module_name) 138 self.results.requested.append(tr_record) 139 140 return test_names 141 142 def RunMoblyModule(self): 143 '''Execute mobly test module.''' 144 # Because mobly and vts uses a similar runner, both will modify 145 # log_path from python logging. The following step is to preserve 146 # log path after mobly test finishes. 147 148 # An alternative way is to start a new python process through shell 149 # command. In that case, test print out needs to be piped. 150 # This will also help avoid log overlapping 151 152 logger = logging.getLogger() 153 logger_path = logger.log_path 154 logging_path = logging.log_path 155 156 try: 157 mobly_test_runner.main(argv=['-c', self.mobly_config_file_path]) 158 finally: 159 logger.log_path = logger_path 160 logging.log_path = logging_path 161 162 def GetMoblyResults(self): 163 '''Get mobly module run results and put in vts results.''' 164 file_handlers = ( 165 (MOBLY_RESULT_YAML_FILE_NAME, self.ParseYamlResults), 166 (MOBLY_RESULT_JSON_FILE_NAME, self.ParseJsonResults), 167 ) 168 169 for pair in file_handlers: 170 file_path = file_util.FindFile(self.mobly_dir, pair[0]) 171 172 if file_path: 173 logging.info('Mobly test yaml result path: %s', file_path) 174 pair[1](file_path) 175 return 176 177 asserts.fail('Mobly test result file not found.') 178 179 def generateAllTests(self): 180 '''Run the mobly test module and parse results.''' 181 #TODO(yuexima): report test names 182 183 self.PrepareConfigFile() 184 self.RunMoblyModule() 185 #TODO(yuexima): check whether DEBUG logs from mobly run are included 186 self.GetMoblyResults() 187 188 def ParseJsonResults(self, result_path): 189 '''Parse mobly test json result. 190 191 Args: 192 result_path: string, result json file path. 193 ''' 194 with open(path, 'r') as f: 195 mobly_summary = json.load(f) 196 197 mobly_results = mobly_summary['Results'] 198 for result in mobly_results: 199 logging.info('Adding result for %s' % result[records.TestResultEnums.RECORD_NAME]) 200 record = records.TestResultRecord(result[records.TestResultEnums.RECORD_NAME]) 201 record.test_class = result[records.TestResultEnums.RECORD_CLASS] 202 record.begin_time = result[records.TestResultEnums.RECORD_BEGIN_TIME] 203 record.end_time = result[records.TestResultEnums.RECORD_END_TIME] 204 record.result = result[records.TestResultEnums.RECORD_RESULT] 205 record.uid = result[records.TestResultEnums.RECORD_UID] 206 record.extras = result[records.TestResultEnums.RECORD_EXTRAS] 207 record.details = result[records.TestResultEnums.RECORD_DETAILS] 208 record.extra_errors = result[records.TestResultEnums.RECORD_EXTRA_ERRORS] 209 210 self.results.addRecord(record) 211 212 def ParseYamlResults(self, result_path): 213 '''Parse mobly test yaml result. 214 215 Args: 216 result_path: string, result yaml file path. 217 ''' 218 with open(result_path, 'r') as stream: 219 try: 220 docs = yaml.load_all(stream) 221 for doc in docs: 222 type = doc.get(RESULT_KEY_TYPE) 223 if type is None: 224 logging.warn( 225 'Mobly result document type unrecognized: %s', doc) 226 continue 227 228 logging.info('Parsing result type: %s', type) 229 230 handler = self.result_handlers.get(type) 231 if handler is None: 232 logging.info('Unknown result type: %s', type) 233 handler = self.HandleSimplePrint 234 235 handler(doc) 236 except yaml.YAMLError as exc: 237 print(exc) 238 239 def HandleRecord(self, doc): 240 '''Handle record result document type. 241 242 Args: 243 doc: dict, result document item 244 ''' 245 logging.info('Adding result for %s' % doc.get(records.TestResultEnums.RECORD_NAME)) 246 record = records.TestResultRecord(doc.get(records.TestResultEnums.RECORD_NAME)) 247 record.test_class = doc.get(records.TestResultEnums.RECORD_CLASS) 248 record.begin_time = doc.get(records.TestResultEnums.RECORD_BEGIN_TIME) 249 record.end_time = doc.get(records.TestResultEnums.RECORD_END_TIME) 250 record.result = doc.get(records.TestResultEnums.RECORD_RESULT) 251 record.uid = doc.get(records.TestResultEnums.RECORD_UID) 252 record.extras = doc.get(records.TestResultEnums.RECORD_EXTRAS) 253 record.details = doc.get(records.TestResultEnums.RECORD_DETAILS) 254 record.extra_errors = doc.get(records.TestResultEnums.RECORD_EXTRA_ERRORS) 255 256 # 'Stacktrace' in yaml result is ignored. 'Stacktrace' is a more 257 # detailed version of record.details when exception is emitted. 258 259 self.results.addRecord(record) 260 261 def HandleSimplePrint(self, doc): 262 '''Simply print result document to log. 263 264 Args: 265 doc: dict, result document item 266 ''' 267 for k, v in doc.items(): 268 logging.info(str(k) + ": " + str(v)) 269 270 def GetTestModuleNames(): 271 '''Returns a list of mobly test module specified in test configuration.''' 272 configs = config_parser.load_test_config_file(sys.argv[1]) 273 reduce_func = lambda x, y: x + y.get(keys.ConfigKeys.MOBLY_TEST_MODULE, []) 274 return reduce(reduce_func, configs, []) 275 276 def ImportTestModules(): 277 '''Dynamically import mobly test modules.''' 278 for module_name in GetTestModuleNames(): 279 module, cls = module_name.rsplit('.', 1) 280 sys.modules['__main__'].__dict__[cls] = getattr( 281 importlib.import_module(module), cls) 282 283 if __name__ == "__main__": 284 ImportTestModules() 285 test_runner.main() 286
