1 import copy 2 import json 3 import logging 4 import re 5 6 import common 7 8 from autotest_lib.client.common_lib import autotemp 9 from autotest_lib.client.common_lib import global_config 10 11 12 # Try importing the essential bug reporting libraries. Chromite and gdata_lib 13 # are useless unless they can import gdata too. 14 try: 15 __import__('chromite') 16 __import__('gdata') 17 except ImportError, e: 18 fundamental_libs = False 19 logging.debug('Will not be able to generate link ' 20 'to the buildbot page when filing bugs. %s', e) 21 else: 22 from chromite.lib import cros_build_lib, gs 23 fundamental_libs = True 24 25 26 # Number of times to retry if a gs command fails. Defaults to 10, 27 # which is far too long given that we already wait on these files 28 # before starting HWTests. 29 _GS_RETRIES = 1 30 31 32 _HTTP_ERROR_THRESHOLD = 400 33 BUG_CONFIG_SECTION = 'BUG_REPORTING' 34 35 # global configurations needed for build artifacts 36 _gs_domain = global_config.global_config.get_config_value( 37 BUG_CONFIG_SECTION, 'gs_domain', default='') 38 _chromeos_image_archive = global_config.global_config.get_config_value( 39 BUG_CONFIG_SECTION, 'chromeos_image_archive', default='') 40 _arg_prefix = global_config.global_config.get_config_value( 41 BUG_CONFIG_SECTION, 'arg_prefix', default='') 42 43 44 # global configurations needed for results log 45 _retrieve_logs_cgi = global_config.global_config.get_config_value( 46 BUG_CONFIG_SECTION, 'retrieve_logs_cgi', default='') 47 _generic_results_bin = global_config.global_config.get_config_value( 48 BUG_CONFIG_SECTION, 'generic_results_bin', default='') 49 _debug_dir = global_config.global_config.get_config_value( 50 BUG_CONFIG_SECTION, 'debug_dir', default='') 51 52 53 # Template for the url used to generate the link to the job 54 _job_view = global_config.global_config.get_config_value( 55 BUG_CONFIG_SECTION, 'job_view', default='') 56 57 58 # gs prefix to perform file like operations (gs://) 59 _gs_file_prefix = global_config.global_config.get_config_value( 60 BUG_CONFIG_SECTION, 'gs_file_prefix', default='') 61 62 63 # global configurations needed for buildbot stages link 64 _buildbot_builders = global_config.global_config.get_config_value( 65 BUG_CONFIG_SECTION, 'buildbot_builders', default='') 66 _build_prefix = global_config.global_config.get_config_value( 67 BUG_CONFIG_SECTION, 'build_prefix', default='') 68 69 _CRBUG_URL = global_config.global_config.get_config_value( 70 BUG_CONFIG_SECTION, 'crbug_url') 71 72 73 WMATRIX_RETRY_URL = global_config.global_config.get_config_value( 74 BUG_CONFIG_SECTION, 'wmatrix_retry_url') 75 WMATRIX_TEST_HISTORY_URL = global_config.global_config.get_config_value( 76 BUG_CONFIG_SECTION, 'wmatrix_test_history_url') 77 78 79 class InvalidBugTemplateException(Exception): 80 """Exception raised when a bug template is not valid, e.g., missing value 81 for essential attributes. 82 """ 83 pass 84 85 86 class BugTemplate(object): 87 """Wrapper class to merge a suite and test bug templates, and do validation. 88 """ 89 90 # Names of expected attributes. 91 EXPECTED_BUG_TEMPLATE_ATTRIBUTES = ['owner', 'labels', 'status', 'title', 92 'cc', 'summary', 'components'] 93 LIST_ATTRIBUTES = ['cc', 'labels'] 94 EMAIL_ATTRIBUTES = ['owner', 'cc'] 95 96 EMAIL_REGEX = re.compile(r'[^@]+@[^@]+\.[^@]+') 97 98 99 def __init__(self, bug_template): 100 """Initialize a BugTemplate object. 101 102 @param bug_template: initial bug template, e.g., bug template from suite 103 control file. 104 """ 105 self.bug_template = self.cleanup_bug_template(bug_template) 106 107 108 @classmethod 109 def validate_bug_template(cls, bug_template): 110 """Verify if a bug template has value for all essential attributes. 111 112 @param bug_template: bug template to be verified. 113 @raise InvalidBugTemplateException: raised when a bug template 114 is invalid, e.g., has missing essential attribute, or any given 115 template is not a dictionary. 116 """ 117 if not type(bug_template) is dict: 118 raise InvalidBugTemplateException('Bug template must be a ' 119 'dictionary.') 120 121 unexpected_keys = [] 122 for key, value in bug_template.iteritems(): 123 if not key in cls.EXPECTED_BUG_TEMPLATE_ATTRIBUTES: 124 raise InvalidBugTemplateException('Key %s is not expected in ' 125 'bug template.' % key) 126 if (key in cls.LIST_ATTRIBUTES and 127 not isinstance(value, list)): 128 raise InvalidBugTemplateException('Value for %s must be a list.' 129 % key) 130 if key in cls.EMAIL_ATTRIBUTES: 131 emails = value if isinstance(value, list) else [value] 132 for email in emails: 133 if not email or not cls.EMAIL_REGEX.match(email): 134 raise InvalidBugTemplateException( 135 'Invalid email address: %s.' % email) 136 137 138 @classmethod 139 def cleanup_bug_template(cls, bug_template): 140 """Remove empty entries in given bug template. 141 142 @param bug_template: bug template to be verified. 143 144 @return: A cleaned up bug template. 145 @raise InvalidBugTemplateException: raised when a bug template 146 is not a dictionary. 147 """ 148 if not type(bug_template) is dict: 149 raise InvalidBugTemplateException('Bug template must be a ' 150 'dictionary.') 151 template = copy.deepcopy(bug_template) 152 # If owner or cc is set but the value is empty or None, remove it from 153 # the template. 154 for email_attribute in cls.EMAIL_ATTRIBUTES: 155 if email_attribute in template: 156 value = template[email_attribute] 157 if isinstance(value, list): 158 template[email_attribute] = [email for email in value 159 if email] 160 if not template[email_attribute]: 161 del(template[email_attribute]) 162 return template 163 164 165 def finalize_bug_template(self, test_template): 166 """Merge test and suite bug templates. 167 168 @param test_template: Bug template from test control file. 169 @return: Merged bug template. 170 171 @raise InvalidBugTemplateException: raised when the merged template is 172 invalid, e.g., has missing essential attribute, or any given 173 template is not a dictionary. 174 """ 175 test_template = self.cleanup_bug_template(test_template) 176 self.validate_bug_template(self.bug_template) 177 self.validate_bug_template(test_template) 178 179 merged_template = test_template 180 merged_template.update((k, v) for k, v in self.bug_template.iteritems() 181 if k not in merged_template) 182 183 # test_template wins for common keys, unless values are list that can be 184 # merged. 185 for key in set(merged_template.keys()).intersection( 186 self.bug_template.keys()): 187 if (type(merged_template[key]) is list and 188 type(self.bug_template[key]) is list): 189 merged_template[key] = (merged_template[key] + 190 self.bug_template[key]) 191 elif not merged_template[key]: 192 merged_template[key] = self.bug_template[key] 193 self.validate_bug_template(merged_template) 194 return merged_template 195 196 197 def link_build_artifacts(build): 198 """Returns a url to build artifacts on google storage. 199 200 @param build: A string, e.g. stout32-release/R30-4433.0.0 201 202 @returns: A url to build artifacts on google storage. 203 204 """ 205 return (_gs_domain + _arg_prefix + 206 _chromeos_image_archive + build) 207 208 209 def link_job(job_id, instance_server=None): 210 """Returns an url to the job on cautotest. 211 212 @param job_id: A string, representing the job id. 213 @param instance_server: The instance server. 214 Eg: cautotest, cautotest-cq, localhost. 215 216 @returns: An url to the job on cautotest. 217 218 """ 219 if not job_id: 220 return 'Job did not run, or was aborted prematurely' 221 if not instance_server: 222 instance_server = global_config.global_config.get_config_value( 223 'SERVER', 'hostname', default='localhost') 224 if 'cautotest' in instance_server: 225 instance_server += '.corp.google.com' 226 return _job_view % (instance_server, job_id) 227 228 229 def _base_results_log(job_id, result_owner, hostname): 230 """Returns the base url of the job's results. 231 232 @param job_id: A string, representing the job id. 233 @param result_owner: A string, representing the onwer of the job. 234 @param hostname: A string, representing the host on which 235 the job has run. 236 237 @returns: The base url of the job's results. 238 239 """ 240 if job_id and result_owner and hostname: 241 path_to_object = '%s-%s/%s' % (job_id, result_owner, 242 hostname) 243 return (_retrieve_logs_cgi + _generic_results_bin + 244 path_to_object) 245 246 247 def link_result_logs(job_id, result_owner, hostname): 248 """Returns a url to test logs on google storage. 249 250 @param job_id: A string, representing the job id. 251 @param result_owner: A string, representing the owner of the job. 252 @param hostname: A string, representing the host on which the 253 jot has run. 254 255 @returns: A url to test logs on google storage. 256 257 """ 258 base_results = _base_results_log(job_id, result_owner, hostname) 259 if base_results: 260 return '%s/%s' % (base_results, _debug_dir) 261 return ('Could not generate results log: the job with id %s, ' 262 'scheduled by: %s on host: %s did not run' % 263 (job_id, result_owner, hostname)) 264 265 266 def link_status_log(job_id, result_owner, hostname): 267 """Returns an url to status log of the job. 268 269 @param job_id: A string, representing the job id. 270 @param result_owner: A string, representing the owner of the job. 271 @param hostname: A string, representing the host on which the 272 jot has run. 273 274 @returns: A url to status log of the job. 275 276 """ 277 base_results = _base_results_log(job_id, result_owner, hostname) 278 if base_results: 279 return '%s/%s' % (base_results, 'status.log') 280 return 'NA' 281 282 283 def _get_metadata_dict(build): 284 """ 285 Get a dictionary of metadata related to this failure. 286 287 Metadata.json is created in the HWTest Archiving stage, if this file 288 isn't found the call to Cat will timeout after the number of retries 289 specified in the GSContext object. If metadata.json exists we parse 290 a json string of it's contents into a dictionary, which we return. 291 292 @param build: A string, e.g. stout32-release/R30-4433.0.0 293 294 @returns: A dictionary with the contents of metadata.json. 295 296 """ 297 if not fundamental_libs: 298 return 299 try: 300 tempdir = autotemp.tempdir() 301 gs_context = gs.GSContext(retries=_GS_RETRIES, 302 cache_dir=tempdir.name) 303 gs_cmd = '%s%s%s/metadata.json' % (_gs_file_prefix, 304 _chromeos_image_archive, 305 build) 306 return json.loads(gs_context.Cat(gs_cmd)) 307 except (cros_build_lib.RunCommandError, gs.GSContextException) as e: 308 logging.debug(e) 309 finally: 310 tempdir.clean() 311 312 313 def link_buildbot_stages(build): 314 """ 315 Link to the buildbot page associated with this run of HWTests. 316 317 @param build: A string, e.g. stout32-release/R30-4433.0.0 318 319 @return: A link to the buildbot stages page, or 'NA' if we cannot glean 320 enough information from metadata.json (or it doesn't exist). 321 """ 322 metadata = _get_metadata_dict(build) 323 if (metadata and 324 metadata.get('builder-name') and 325 metadata.get('build-number')): 326 327 return ('%s%s/builds/%s' % 328 (_buildbot_builders, 329 metadata.get('builder-name'), 330 metadata.get('build-number'))).replace(' ', '%20') 331 return 'NA' 332 333 334 def link_retry_url(test_name): 335 """Link to the wmatrix retry stats page for this test. 336 337 @param test_name: Test we want to search the retry stats page for. 338 339 @return: A link to the wmatrix retry stats dashboard for this test. 340 """ 341 return WMATRIX_RETRY_URL % test_name 342 343 344 def link_test_history(test_name): 345 """Link to the wmatrix test history page for this test. 346 347 @param test_name: Test we want to search the test history for. 348 349 @return: A link to the wmatrix test history page for this test. 350 """ 351 return WMATRIX_TEST_HISTORY_URL % test_name 352 353 354 def link_crbug(bug_id): 355 """Generate a bug link for the given bug_id. 356 357 @param bug_id: The id of the bug. 358 @return: A link, eg: https://crbug.com/<bug_id>. 359 """ 360 return _CRBUG_URL % (bug_id,) 361
