1 # 2 # Copyright (C) 2017 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # 16 17 import base64 18 import getpass 19 import logging 20 import os 21 import socket 22 import time 23 24 from vts.proto import VtsReportMessage_pb2 as ReportMsg 25 from vts.runners.host import keys 26 from vts.utils.python.web import dashboard_rest_client 27 from vts.utils.python.web import feature_utils 28 29 _PROFILING_POINTS = "profiling_points" 30 31 32 class WebFeature(feature_utils.Feature): 33 """Feature object for web functionality. 34 35 Attributes: 36 enabled: boolean, True if systrace is enabled, False otherwise 37 report_msg: TestReportMessage, Proto summarizing the test run 38 current_test_report_msg: TestCaseReportMessage, Proto summarizing the current test case 39 rest_client: DashboardRestClient, client to which data will be posted 40 """ 41 42 _TOGGLE_PARAM = keys.ConfigKeys.IKEY_ENABLE_WEB 43 _REQUIRED_PARAMS = [ 44 keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND, 45 keys.ConfigKeys.IKEY_SERVICE_JSON_PATH, 46 keys.ConfigKeys.KEY_TESTBED_NAME, keys.ConfigKeys.IKEY_BUILD, 47 keys.ConfigKeys.IKEY_ANDROID_DEVICE, keys.ConfigKeys.IKEY_ABI_NAME, 48 keys.ConfigKeys.IKEY_ABI_BITNESS 49 ] 50 _OPTIONAL_PARAMS = [keys.ConfigKeys.RUN_AS_VTS_SELFTEST] 51 52 def __init__(self, user_params): 53 """Initializes the web feature. 54 55 Parses the arguments and initializes the web functionality. 56 57 Args: 58 user_params: A dictionary from parameter name (String) to parameter value. 59 """ 60 self.ParseParameters( 61 toggle_param_name=self._TOGGLE_PARAM, 62 required_param_names=self._REQUIRED_PARAMS, 63 optional_param_names=self._OPTIONAL_PARAMS, 64 user_params=user_params) 65 if not self.enabled: 66 return 67 68 # Initialize the dashboard client 69 post_cmd = getattr(self, keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND) 70 service_json_path = str( 71 getattr(self, keys.ConfigKeys.IKEY_SERVICE_JSON_PATH)) 72 self.rest_client = dashboard_rest_client.DashboardRestClient( 73 post_cmd, service_json_path) 74 if not self.rest_client.Initialize(): 75 self.enabled = False 76 77 self.report_msg = ReportMsg.TestReportMessage() 78 self.report_msg.test = str( 79 getattr(self, keys.ConfigKeys.KEY_TESTBED_NAME)) 80 self.report_msg.test_type = ReportMsg.VTS_HOST_DRIVEN_STRUCTURAL 81 self.report_msg.start_timestamp = feature_utils.GetTimestamp() 82 self.report_msg.host_info.hostname = socket.gethostname() 83 84 android_devices = getattr(self, keys.ConfigKeys.IKEY_ANDROID_DEVICE, 85 None) 86 if not android_devices or not isinstance(android_devices, list): 87 logging.warn("android device information not available") 88 return 89 90 for device_spec in android_devices: 91 dev_info = self.report_msg.device_info.add() 92 for elem in [ 93 keys.ConfigKeys.IKEY_PRODUCT_TYPE, 94 keys.ConfigKeys.IKEY_PRODUCT_VARIANT, 95 keys.ConfigKeys.IKEY_BUILD_FLAVOR, 96 keys.ConfigKeys.IKEY_BUILD_ID, keys.ConfigKeys.IKEY_BRANCH, 97 keys.ConfigKeys.IKEY_BUILD_ALIAS, 98 keys.ConfigKeys.IKEY_API_LEVEL, keys.ConfigKeys.IKEY_SERIAL 99 ]: 100 if elem in device_spec: 101 setattr(dev_info, elem, str(device_spec[elem])) 102 # TODO: get abi information differently for multi-device support. 103 setattr(dev_info, keys.ConfigKeys.IKEY_ABI_NAME, 104 str(getattr(self, keys.ConfigKeys.IKEY_ABI_NAME))) 105 setattr(dev_info, keys.ConfigKeys.IKEY_ABI_BITNESS, 106 str(getattr(self, keys.ConfigKeys.IKEY_ABI_BITNESS))) 107 108 def SetTestResult(self, result=None): 109 """Set the current test case result to the provided result. 110 111 If None is provided as a result, the current test report will be cleared, which results 112 in a silent skip. 113 114 Requires the feature to be enabled; no-op otherwise. 115 116 Args: 117 result: ReportMsg.TestCaseResult, the result of the current test or None. 118 """ 119 if not self.enabled: 120 return 121 122 if not result: 123 self.report_msg.test_case.remove(self.current_test_report_msg) 124 self.current_test_report_msg = None 125 else: 126 self.current_test_report_msg.test_result = result 127 128 def AddTestReport(self, test_name): 129 """Creates a report for the specified test. 130 131 Requires the feature to be enabled; no-op otherwise. 132 133 Args: 134 test_name: String, the name of the test 135 """ 136 if not self.enabled: 137 return 138 self.current_test_report_msg = self.report_msg.test_case.add() 139 self.current_test_report_msg.name = test_name 140 self.current_test_report_msg.start_timestamp = feature_utils.GetTimestamp( 141 ) 142 143 def AddCoverageReport(self, 144 coverage_vec, 145 src_file_path, 146 git_project_name, 147 git_project_path, 148 revision, 149 covered_count, 150 line_count, 151 isGlobal=True): 152 """Adds a coverage report to the VtsReportMessage. 153 154 Processes the source information, git project information, and processed 155 coverage information and stores it into a CoverageReportMessage within the 156 report message. 157 158 Args: 159 coverage_vec: list, list of coverage counts (int) for each line 160 src_file_path: the path to the original source file 161 git_project_name: the name of the git project containing the source 162 git_project_path: the path from the root to the git project 163 revision: the commit hash identifying the source code that was used to 164 build a device image 165 covered_count: int, number of lines covered 166 line_count: int, total number of lines 167 isGlobal: boolean, True if the coverage data is for the entire test, False if only for 168 the current test case. 169 """ 170 if not self.enabled: 171 return 172 173 if isGlobal: 174 report = self.report_msg 175 else: 176 report = self.current_test_report_msg 177 178 coverage = report.coverage.add() 179 coverage.total_line_count = line_count 180 coverage.covered_line_count = covered_count 181 coverage.line_coverage_vector.extend(coverage_vec) 182 183 src_file_path = os.path.relpath(src_file_path, git_project_path) 184 coverage.file_path = src_file_path 185 coverage.revision = revision 186 coverage.project_name = git_project_name 187 188 def AddProfilingDataTimestamp( 189 self, 190 name, 191 start_timestamp, 192 end_timestamp, 193 x_axis_label="Latency (nano secs)", 194 y_axis_label="Frequency", 195 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 196 """Adds the timestamp profiling data to the web DB. 197 198 Requires the feature to be enabled; no-op otherwise. 199 200 Args: 201 name: string, profiling point name. 202 start_timestamp: long, nanoseconds start time. 203 end_timestamp: long, nanoseconds end time. 204 x-axis_label: string, the x-axis label title for a graph plot. 205 y-axis_label: string, the y-axis label title for a graph plot. 206 regression_mode: specifies the direction of change which indicates 207 performance regression. 208 """ 209 if not self.enabled: 210 return 211 212 if not hasattr(self, _PROFILING_POINTS): 213 setattr(self, _PROFILING_POINTS, set()) 214 215 if name in getattr(self, _PROFILING_POINTS): 216 logging.error("profiling point %s is already active.", name) 217 return 218 219 getattr(self, _PROFILING_POINTS).add(name) 220 profiling_msg = self.report_msg.profiling.add() 221 profiling_msg.name = name 222 profiling_msg.type = ReportMsg.VTS_PROFILING_TYPE_TIMESTAMP 223 profiling_msg.regression_mode = regression_mode 224 profiling_msg.start_timestamp = start_timestamp 225 profiling_msg.end_timestamp = end_timestamp 226 profiling_msg.x_axis_label = x_axis_label 227 profiling_msg.y_axis_label = y_axis_label 228 229 def AddProfilingDataVector( 230 self, 231 name, 232 labels, 233 values, 234 data_type, 235 options=[], 236 x_axis_label="x-axis", 237 y_axis_label="y-axis", 238 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 239 """Adds the vector profiling data in order to upload to the web DB. 240 241 Requires the feature to be enabled; no-op otherwise. 242 243 Args: 244 name: string, profiling point name. 245 labels: a list or set of labels. 246 values: a list or set of values where each value is an integer. 247 data_type: profiling data type. 248 options: a set of options. 249 x-axis_label: string, the x-axis label title for a graph plot. 250 y-axis_label: string, the y-axis label title for a graph plot. 251 regression_mode: specifies the direction of change which indicates 252 performance regression. 253 """ 254 if not self.enabled: 255 return 256 257 if not hasattr(self, _PROFILING_POINTS): 258 setattr(self, _PROFILING_POINTS, set()) 259 260 if name in getattr(self, _PROFILING_POINTS): 261 logging.error("profiling point %s is already active.", name) 262 return 263 264 getattr(self, _PROFILING_POINTS).add(name) 265 profiling_msg = self.report_msg.profiling.add() 266 profiling_msg.name = name 267 profiling_msg.type = data_type 268 profiling_msg.regression_mode = regression_mode 269 if labels: 270 profiling_msg.label.extend(labels) 271 profiling_msg.value.extend(values) 272 profiling_msg.x_axis_label = x_axis_label 273 profiling_msg.y_axis_label = y_axis_label 274 profiling_msg.options.extend(options) 275 276 def AddProfilingDataLabeledVector( 277 self, 278 name, 279 labels, 280 values, 281 options=[], 282 x_axis_label="x-axis", 283 y_axis_label="y-axis", 284 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 285 """Adds the labeled vector profiling data in order to upload to the web DB. 286 287 Requires the feature to be enabled; no-op otherwise. 288 289 Args: 290 name: string, profiling point name. 291 labels: a list or set of labels. 292 values: a list or set of values where each value is an integer. 293 options: a set of options. 294 x-axis_label: string, the x-axis label title for a graph plot. 295 y-axis_label: string, the y-axis label title for a graph plot. 296 regression_mode: specifies the direction of change which indicates 297 performance regression. 298 """ 299 self.AddProfilingDataVector( 300 name, labels, values, ReportMsg.VTS_PROFILING_TYPE_LABELED_VECTOR, 301 options, x_axis_label, y_axis_label, regression_mode) 302 303 def AddProfilingDataUnlabeledVector( 304 self, 305 name, 306 values, 307 options=[], 308 x_axis_label="x-axis", 309 y_axis_label="y-axis", 310 regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): 311 """Adds the unlabeled vector profiling data in order to upload to the web DB. 312 313 Requires the feature to be enabled; no-op otherwise. 314 315 Args: 316 name: string, profiling point name. 317 values: a list or set of values where each value is an integer. 318 options: a set of options. 319 x-axis_label: string, the x-axis label title for a graph plot. 320 y-axis_label: string, the y-axis label title for a graph plot. 321 regression_mode: specifies the direction of change which indicates 322 performance regression. 323 """ 324 self.AddProfilingDataVector( 325 name, None, values, ReportMsg.VTS_PROFILING_TYPE_UNLABELED_VECTOR, 326 options, x_axis_label, y_axis_label, regression_mode) 327 328 def AddSystraceUrl(self, url): 329 """Creates a systrace report message with a systrace URL. 330 331 Adds a systrace report to the current test case report and supplies the 332 url to the systrace report. 333 334 Requires the feature to be enabled; no-op otherwise. 335 336 Args: 337 url: String, the url of the systrace report. 338 """ 339 if not self.enabled: 340 return 341 systrace_msg = self.current_test_report_msg.systrace.add() 342 systrace_msg.url.append(url) 343 344 def AddLogUrls(self, urls): 345 """Creates a log message with log file URLs. 346 347 Adds a log message to the current test module report and supplies the 348 url to the log files. 349 350 Requires the feature to be enabled; no-op otherwise. 351 352 Args: 353 urls: list of string, the URLs of the logs. 354 """ 355 if not self.enabled or urls is None: 356 return 357 358 for url in urls: 359 log_msg = self.report_msg.log.add() 360 log_msg.url = url 361 log_msg.name = os.path.basename(url) 362 363 def GetTestModuleKeys(self): 364 """Returns the test module name and start timestamp. 365 366 Those two values can be used to find the corresponding entry 367 in a used nosql database without having to lock all the data 368 (which is infesiable) thus are essential for strong consistency. 369 """ 370 return self.report_msg.test, self.report_msg.start_timestamp 371 372 def GenerateReportMessage(self, requested, executed): 373 """Uploads the result to the web service. 374 375 Requires the feature to be enabled; no-op otherwise. 376 377 Args: 378 requested: list, A list of test case records requested to run 379 executed: list, A list of test case records that were executed 380 381 Returns: 382 binary string, serialized report message. 383 None if web is not enabled. 384 """ 385 if not self.enabled: 386 return None 387 388 # Handle case when runner fails, tests aren't executed 389 if (not getattr(self, keys.ConfigKeys.RUN_AS_VTS_SELFTEST, False) 390 and executed and executed[-1].test_name == "setup_class"): 391 # Test failed during setup, all tests were not executed 392 start_index = 0 393 else: 394 # Runner was aborted. Remaining tests weren't executed 395 start_index = len(executed) 396 397 for test in requested[start_index:]: 398 msg = self.report_msg.test_case.add() 399 msg.name = test.test_name 400 msg.start_timestamp = feature_utils.GetTimestamp() 401 msg.end_timestamp = msg.start_timestamp 402 msg.test_result = ReportMsg.TEST_CASE_RESULT_FAIL 403 404 self.report_msg.end_timestamp = feature_utils.GetTimestamp() 405 406 build = getattr(self, keys.ConfigKeys.IKEY_BUILD) 407 if keys.ConfigKeys.IKEY_BUILD_ID in build: 408 build_id = str(build[keys.ConfigKeys.IKEY_BUILD_ID]) 409 self.report_msg.build_info.id = build_id 410 411 logging.info("_tearDownClass hook: start (username: %s)", 412 getpass.getuser()) 413 414 if len(self.report_msg.test_case) == 0: 415 logging.info("_tearDownClass hook: skip uploading (no test case)") 416 return '' 417 418 post_msg = ReportMsg.DashboardPostMessage() 419 post_msg.test_report.extend([self.report_msg]) 420 421 self.rest_client.AddAuthToken(post_msg) 422 423 message_b = base64.b64encode(post_msg.SerializeToString()) 424 425 logging.info('Result proto message generated. size: %s', 426 len(message_b)) 427 428 logging.info("_tearDownClass hook: status upload time stamp %s", 429 str(self.report_msg.start_timestamp)) 430 431 return message_b 432
