1 import copy 2 import json 3 import logging 4 import re 5 6 import common 7 8 from autotest_lib.client.common_lib import autotemp 9 from autotest_lib.client.common_lib import global_config 10 11 12 # Try importing the essential bug reporting libraries. Chromite and gdata_lib 13 # are useless unless they can import gdata too. 14 try: 15 __import__('chromite') 16 __import__('gdata') 17 except ImportError, e: 18 fundamental_libs = False 19 logging.debug('Will not be able to generate link ' 20 'to the buildbot page when filing bugs. %s', e) 21 else: 22 from chromite.lib import cros_build_lib, gs 23 fundamental_libs = True 24 25 26 # Number of times to retry if a gs command fails. Defaults to 10, 27 # which is far too long given that we already wait on these files 28 # before starting HWTests. 29 _GS_RETRIES = 1 30 31 32 _HTTP_ERROR_THRESHOLD = 400 33 BUG_CONFIG_SECTION = 'BUG_REPORTING' 34 35 # global configurations needed for build artifacts 36 _gs_domain = global_config.global_config.get_config_value( 37 BUG_CONFIG_SECTION, 'gs_domain', default='') 38 _chromeos_image_archive = global_config.global_config.get_config_value( 39 BUG_CONFIG_SECTION, 'chromeos_image_archive', default='') 40 _arg_prefix = global_config.global_config.get_config_value( 41 BUG_CONFIG_SECTION, 'arg_prefix', default='') 42 43 44 # global configurations needed for results log 45 _retrieve_logs_cgi = global_config.global_config.get_config_value( 46 BUG_CONFIG_SECTION, 'retrieve_logs_cgi', default='') 47 _generic_results_bin = global_config.global_config.get_config_value( 48 BUG_CONFIG_SECTION, 'generic_results_bin', default='') 49 _debug_dir = global_config.global_config.get_config_value( 50 BUG_CONFIG_SECTION, 'debug_dir', default='') 51 52 53 # Template for the url used to generate the link to the job 54 _job_view = global_config.global_config.get_config_value( 55 BUG_CONFIG_SECTION, 'job_view', default='') 56 57 58 # gs prefix to perform file like operations (gs://) 59 _gs_file_prefix = global_config.global_config.get_config_value( 60 BUG_CONFIG_SECTION, 'gs_file_prefix', default='') 61 62 63 # global configurations needed for buildbot stages link 64 _buildbot_builders = global_config.global_config.get_config_value( 65 BUG_CONFIG_SECTION, 'buildbot_builders', default='') 66 _build_prefix = global_config.global_config.get_config_value( 67 BUG_CONFIG_SECTION, 'build_prefix', default='') 68 69 70 WMATRIX_RETRY_URL = global_config.global_config.get_config_value( 71 BUG_CONFIG_SECTION, 'wmatrix_retry_url') 72 73 74 class InvalidBugTemplateException(Exception): 75 """Exception raised when a bug template is not valid, e.g., missing value 76 for essential attributes. 77 """ 78 pass 79 80 81 class BugTemplate(object): 82 """Wrapper class to merge a suite and test bug templates, and do validation. 83 """ 84 85 # Names of expected attributes. 86 EXPECTED_BUG_TEMPLATE_ATTRIBUTES = ['owner', 'labels', 'status', 'title', 87 'cc', 'summary'] 88 LIST_ATTRIBUTES = ['cc', 'labels'] 89 EMAIL_ATTRIBUTES = ['owner', 'cc'] 90 91 EMAIL_REGEX = re.compile(r'[^@]+@[^@]+\.[^@]+') 92 93 94 def __init__(self, bug_template): 95 """Initialize a BugTemplate object. 96 97 @param bug_template: initial bug template, e.g., bug template from suite 98 control file. 99 """ 100 self.bug_template = self.cleanup_bug_template(bug_template) 101 102 103 @classmethod 104 def validate_bug_template(cls, bug_template): 105 """Verify if a bug template has value for all essential attributes. 106 107 @param bug_template: bug template to be verified. 108 @raise InvalidBugTemplateException: raised when a bug template 109 is invalid, e.g., has missing essential attribute, or any given 110 template is not a dictionary. 111 """ 112 if not type(bug_template) is dict: 113 raise InvalidBugTemplateException('Bug template must be a ' 114 'dictionary.') 115 116 unexpected_keys = [] 117 for key, value in bug_template.iteritems(): 118 if not key in cls.EXPECTED_BUG_TEMPLATE_ATTRIBUTES: 119 raise InvalidBugTemplateException('Key %s is not expected in ' 120 'bug template.' % key) 121 if (key in cls.LIST_ATTRIBUTES and 122 not isinstance(value, list)): 123 raise InvalidBugTemplateException('Value for %s must be a list.' 124 % key) 125 if key in cls.EMAIL_ATTRIBUTES: 126 emails = value if isinstance(value, list) else [value] 127 for email in emails: 128 if not email or not cls.EMAIL_REGEX.match(email): 129 raise InvalidBugTemplateException( 130 'Invalid email address: %s.' % email) 131 132 133 @classmethod 134 def cleanup_bug_template(cls, bug_template): 135 """Remove empty entries in given bug template. 136 137 @param bug_template: bug template to be verified. 138 139 @return: A cleaned up bug template. 140 @raise InvalidBugTemplateException: raised when a bug template 141 is not a dictionary. 142 """ 143 if not type(bug_template) is dict: 144 raise InvalidBugTemplateException('Bug template must be a ' 145 'dictionary.') 146 template = copy.deepcopy(bug_template) 147 # If owner or cc is set but the value is empty or None, remove it from 148 # the template. 149 for email_attribute in cls.EMAIL_ATTRIBUTES: 150 if email_attribute in template: 151 value = template[email_attribute] 152 if isinstance(value, list): 153 template[email_attribute] = [email for email in value 154 if email] 155 if not template[email_attribute]: 156 del(template[email_attribute]) 157 return template 158 159 160 def finalize_bug_template(self, test_template): 161 """Merge test and suite bug templates. 162 163 @param test_template: Bug template from test control file. 164 @return: Merged bug template. 165 166 @raise InvalidBugTemplateException: raised when the merged template is 167 invalid, e.g., has missing essential attribute, or any given 168 template is not a dictionary. 169 """ 170 test_template = self.cleanup_bug_template(test_template) 171 self.validate_bug_template(self.bug_template) 172 self.validate_bug_template(test_template) 173 174 merged_template = test_template 175 merged_template.update((k, v) for k, v in self.bug_template.iteritems() 176 if k not in merged_template) 177 178 # test_template wins for common keys, unless values are list that can be 179 # merged. 180 for key in set(merged_template.keys()).intersection( 181 self.bug_template.keys()): 182 if (type(merged_template[key]) is list and 183 type(self.bug_template[key]) is list): 184 merged_template[key] = (merged_template[key] + 185 self.bug_template[key]) 186 elif not merged_template[key]: 187 merged_template[key] = self.bug_template[key] 188 self.validate_bug_template(merged_template) 189 return merged_template 190 191 192 def link_build_artifacts(build): 193 """Returns a url to build artifacts on google storage. 194 195 @param build: A string, e.g. stout32-release/R30-4433.0.0 196 197 @returns: A url to build artifacts on google storage. 198 199 """ 200 return (_gs_domain + _arg_prefix + 201 _chromeos_image_archive + build) 202 203 204 def link_job(job_id, instance_server=None): 205 """Returns an url to the job on cautotest. 206 207 @param job_id: A string, representing the job id. 208 @param instance_server: The instance server. 209 Eg: cautotest, cautotest-cq, localhost. 210 211 @returns: An url to the job on cautotest. 212 213 """ 214 if not job_id: 215 return 'Job did not run, or was aborted prematurely' 216 if not instance_server: 217 instance_server = global_config.global_config.get_config_value( 218 'SERVER', 'hostname', default='localhost') 219 if 'cautotest' in instance_server: 220 instance_server += '.corp.google.com' 221 return _job_view % (instance_server, job_id) 222 223 224 def _base_results_log(job_id, result_owner, hostname): 225 """Returns the base url of the job's results. 226 227 @param job_id: A string, representing the job id. 228 @param result_owner: A string, representing the onwer of the job. 229 @param hostname: A string, representing the host on which 230 the job has run. 231 232 @returns: The base url of the job's results. 233 234 """ 235 if job_id and result_owner and hostname: 236 path_to_object = '%s-%s/%s' % (job_id, result_owner, 237 hostname) 238 return (_retrieve_logs_cgi + _generic_results_bin + 239 path_to_object) 240 241 242 def link_result_logs(job_id, result_owner, hostname): 243 """Returns a url to test logs on google storage. 244 245 @param job_id: A string, representing the job id. 246 @param result_owner: A string, representing the owner of the job. 247 @param hostname: A string, representing the host on which the 248 jot has run. 249 250 @returns: A url to test logs on google storage. 251 252 """ 253 base_results = _base_results_log(job_id, result_owner, hostname) 254 if base_results: 255 return '%s/%s' % (base_results, _debug_dir) 256 return ('Could not generate results log: the job with id %s, ' 257 'scheduled by: %s on host: %s did not run' % 258 (job_id, result_owner, hostname)) 259 260 261 def link_status_log(job_id, result_owner, hostname): 262 """Returns an url to status log of the job. 263 264 @param job_id: A string, representing the job id. 265 @param result_owner: A string, representing the owner of the job. 266 @param hostname: A string, representing the host on which the 267 jot has run. 268 269 @returns: A url to status log of the job. 270 271 """ 272 base_results = _base_results_log(job_id, result_owner, hostname) 273 if base_results: 274 return '%s/%s' % (base_results, 'status.log') 275 return 'NA' 276 277 278 def _get_metadata_dict(build): 279 """ 280 Get a dictionary of metadata related to this failure. 281 282 Metadata.json is created in the HWTest Archiving stage, if this file 283 isn't found the call to Cat will timeout after the number of retries 284 specified in the GSContext object. If metadata.json exists we parse 285 a json string of it's contents into a dictionary, which we return. 286 287 @param build: A string, e.g. stout32-release/R30-4433.0.0 288 289 @returns: A dictionary with the contents of metadata.json. 290 291 """ 292 if not fundamental_libs: 293 return 294 try: 295 tempdir = autotemp.tempdir() 296 gs_context = gs.GSContext(retries=_GS_RETRIES, 297 cache_dir=tempdir.name) 298 gs_cmd = '%s%s%s/metadata.json' % (_gs_file_prefix, 299 _chromeos_image_archive, 300 build) 301 return json.loads(gs_context.Cat(gs_cmd)) 302 except (cros_build_lib.RunCommandError, gs.GSContextException) as e: 303 logging.debug(e) 304 finally: 305 tempdir.clean() 306 307 308 def link_buildbot_stages(build): 309 """ 310 Link to the buildbot page associated with this run of HWTests. 311 312 @param build: A string, e.g. stout32-release/R30-4433.0.0 313 314 @return: A link to the buildbot stages page, or 'NA' if we cannot glean 315 enough information from metadata.json (or it doesn't exist). 316 """ 317 metadata = _get_metadata_dict(build) 318 if (metadata and 319 metadata.get('builder-name') and 320 metadata.get('build-number')): 321 322 return ('%s%s/builds/%s' % 323 (_buildbot_builders, 324 metadata.get('builder-name'), 325 metadata.get('build-number'))).replace(' ', '%20') 326 return 'NA' 327 328 329 def link_retry_url(test_name): 330 """Link to the wmatrix retry stats page for this test. 331 332 @param test_name: Test we want to search the retry stats page for. 333 334 @return: A link to the wmatrix retry stats dashboard for this test. 335 """ 336 return WMATRIX_RETRY_URL % test_name
