1 #!/usr/bin/python 2 3 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 import logging 7 8 import common 9 10 import httplib 11 import httplib2 12 from autotest_lib.server.cros.dynamic_suite import constants 13 from chromite.lib import gdata_lib 14 15 try: 16 from apiclient.discovery import build as apiclient_build 17 from apiclient import errors as apiclient_errors 18 from oauth2client import file as oauth_client_fileio 19 except ImportError as e: 20 apiclient_build = None 21 logging.debug("API client for bug filing disabled. %s", e) 22 23 24 class ProjectHostingApiException(Exception): 25 """ 26 Raised when an api call fails, since the actual 27 HTTP error can be cryptic. 28 """ 29 30 31 class BaseIssue(gdata_lib.Issue): 32 """Base issue class with the minimum data to describe a tracker bug. 33 """ 34 def __init__(self, t_issue): 35 kwargs = {} 36 kwargs.update((keys, t_issue.get(keys)) 37 for keys in gdata_lib.Issue.SlotDefaults.keys()) 38 super(BaseIssue, self).__init__(**kwargs) 39 40 41 class Issue(BaseIssue): 42 """ 43 Class representing an Issue and it's related metadata. 44 """ 45 def __init__(self, t_issue): 46 """ 47 Initialize |self| from tracker issue |t_issue| 48 49 @param t_issue: The base issue we want to use to populate 50 the member variables of this object. 51 52 @raises ProjectHostingApiException: If the tracker issue doesn't 53 contain all expected fields needed to create a complete issue. 54 """ 55 super(Issue, self).__init__(t_issue) 56 57 try: 58 # The value keyed under 'summary' in the tracker issue 59 # is, unfortunately, not the summary but the title. The 60 # actual summary is the update at index 0. 61 self.summary = t_issue.get('updates')[0] 62 self.comments = t_issue.get('updates')[1:] 63 64 # open or closed statuses are classified according to labels like 65 # unconfirmed, verified, fixed etc just like through the front end. 66 self.state = t_issue.get(constants.ISSUE_STATE) 67 self.merged_into = None 68 if (t_issue.get(constants.ISSUE_STATUS) 69 == constants.ISSUE_DUPLICATE and 70 constants.ISSUE_MERGEDINTO in t_issue): 71 parent_issue_dict = t_issue.get(constants.ISSUE_MERGEDINTO) 72 self.merged_into = parent_issue_dict.get('issueId') 73 except KeyError as e: 74 raise ProjectHostingApiException('Cannot create a ' 75 'complete issue %s, tracker issue: %s' % (e, t_issue)) 76 77 78 class ProjectHostingApiClient(): 79 """ 80 Client class for interaction with the project hosting api. 81 """ 82 83 # Maximum number of results we would like when querying the tracker. 84 _max_results_for_issue = 50 85 _start_index = 1 86 87 88 def __init__(self, oauth_credentials, project_name): 89 if apiclient_build is None: 90 raise ProjectHostingApiException('Cannot get apiclient library.') 91 92 if not oauth_credentials: 93 raise ProjectHostingApiException('No oauth_credentials is provided.') 94 95 storage = oauth_client_fileio.Storage(oauth_credentials) 96 credentials = storage.get() 97 if credentials is None or credentials.invalid: 98 raise ProjectHostingApiException('Invalid credentials for Project ' 99 'Hosting api. Cannot file bugs.') 100 101 http = credentials.authorize(httplib2.Http()) 102 try: 103 self._codesite_service = apiclient_build('projecthosting', 104 'v2', http=http) 105 except (apiclient_errors.Error, httplib2.HttpLib2Error, 106 httplib.BadStatusLine) as e: 107 raise ProjectHostingApiException(str(e)) 108 self._project_name = project_name 109 110 111 def _execute_request(self, request): 112 """ 113 Executes an api request. 114 115 @param request: An apiclient.http.HttpRequest object representing the 116 request to be executed. 117 @raises: ProjectHostingApiException if we fail to execute the request. 118 This could happen if we receive an http response that is not a 119 2xx, or if the http object itself encounters an error. 120 121 @return: A deserialized object model of the response body returned for 122 the request. 123 """ 124 try: 125 return request.execute() 126 except (apiclient_errors.Error, httplib2.HttpLib2Error, 127 httplib.BadStatusLine) as e: 128 msg = 'Unable to execute your request: %s' 129 raise ProjectHostingApiException(msg % e) 130 131 132 def _get_field(self, field): 133 """ 134 Gets a field from the project. 135 136 This method directly queries the project hosting API using bugdroids1's, 137 api key. 138 139 @param field: A selector, which corresponds loosely to a field in the 140 new bug description of the crosbug frontend. 141 @raises: ProjectHostingApiException, if the request execution fails. 142 143 @return: A json formatted python dict of the specified field's options, 144 or None if we can't find the api library. This dictionary 145 represents the javascript literal used by the front end tracker 146 and can hold multiple filds. 147 148 The returned dictionary follows a template, but it's structure 149 is only loosely defined as it needs to match whatever the front 150 end describes via javascript. 151 For a new issue interface which looks like: 152 153 field 1: text box 154 drop down: predefined value 1 = description 155 predefined value 2 = description 156 field 2: text box 157 similar structure as field 1 158 159 you will get a dictionary like: 160 { 161 'field name 1': { 162 'project realted config': 'config value' 163 'property': [ 164 {predefined value for property 1, description}, 165 {predefined value for property 2, description} 166 ] 167 }, 168 169 'field name 2': { 170 similar structure 171 } 172 ... 173 } 174 """ 175 project = self._codesite_service.projects() 176 request = project.get(projectId=self._project_name, 177 fields=field) 178 return self._execute_request(request) 179 180 181 def _list_updates(self, issue_id): 182 """ 183 Retrieve all updates for a given issue including comments, changes to 184 it's labels, status etc. The first element in the dictionary returned 185 by this method, is by default, the 0th update on the bug; which is the 186 entry that created it. All the text in a given update is keyed as 187 'content', and updates that contain no text, eg: a change to the status 188 of a bug, will contain the emtpy string instead. 189 190 @param issue_id: The id of the issue we want detailed information on. 191 @raises: ProjectHostingApiException, if the request execution fails. 192 193 @return: A json formatted python dict that has an entry for each update 194 performed on this issue. 195 """ 196 issue_comments = self._codesite_service.issues().comments() 197 request = issue_comments.list(projectId=self._project_name, 198 issueId=issue_id, 199 maxResults=self._max_results_for_issue) 200 return self._execute_request(request) 201 202 203 def _get_issue(self, issue_id): 204 """ 205 Gets an issue given it's id. 206 207 @param issue_id: A string representing the issue id. 208 @raises: ProjectHostingApiException, if failed to get the issue. 209 210 @return: A json formatted python dict that has the issue content. 211 """ 212 issues = self._codesite_service.issues() 213 try: 214 request = issues.get(projectId=self._project_name, 215 issueId=issue_id) 216 except TypeError as e: 217 raise ProjectHostingApiException( 218 'Unable to get issue %s from project %s: %s' % 219 (issue_id, self._project_name, str(e))) 220 return self._execute_request(request) 221 222 223 def set_max_results(self, max_results): 224 """Set the max results to return. 225 226 @param max_results: An integer representing the maximum number of 227 matching results to return per query. 228 """ 229 self._max_results_for_issue = max_results 230 231 232 def set_start_index(self, start_index): 233 """Set the start index, for paging. 234 235 @param start_index: The new start index to use. 236 """ 237 self._start_index = start_index 238 239 240 def list_issues(self, **kwargs): 241 """ 242 List issues containing the search marker. This method will only list 243 the summary, title and id of an issue, though it searches through the 244 comments. Eg: if we're searching for the marker '123', issues that 245 contain a comment of '123' will appear in the output, but the string 246 '123' itself may not, because the output only contains issue summaries. 247 248 @param kwargs: 249 q: The anchor string used in the search. 250 can: a string representing the search space that is passed to the 251 google api, can be 'all', 'new', 'open', 'owned', 'reported', 252 'starred', or 'to-verify', defaults to 'all'. 253 label: A string representing a single label to match. 254 255 @return: A json formatted python dict of all matching issues. 256 257 @raises: ProjectHostingApiException, if the request execution fails. 258 """ 259 issues = self._codesite_service.issues() 260 261 # Asking for issues with None or '' labels will restrict the query 262 # to those issues without labels. 263 if not kwargs['label']: 264 del kwargs['label'] 265 266 request = issues.list(projectId=self._project_name, 267 startIndex=self._start_index, 268 maxResults=self._max_results_for_issue, 269 **kwargs) 270 return self._execute_request(request) 271 272 273 def _get_property_values(self, prop_dict): 274 """ 275 Searches a dictionary as returned by _get_field for property lists, 276 then returns each value in the list. Effectively this gives us 277 all the accepted values for a property. For example, in crosbug, 278 'properties' map to things like Status, Labels, Owner etc, each of these 279 will have a list within the issuesConfig dict. 280 281 @param prop_dict: dictionary which contains a list of properties. 282 @yield: each value in a property list. This can be a dict or any other 283 type of datastructure, the caller is responsible for handling 284 it correctly. 285 """ 286 for name, property in prop_dict.iteritems(): 287 if isinstance(property, list): 288 for values in property: 289 yield values 290 291 292 def _get_cros_labels(self, prop_dict): 293 """ 294 Helper function to isolate labels from the labels dictionary. This 295 dictionary is of the form: 296 { 297 "label": "Cr-OS-foo", 298 "description": "description" 299 }, 300 And maps to the frontend like so: 301 Labels: Cr-??? 302 Cr-OS-foo = description 303 where Cr-OS-foo is a conveniently predefined value for Label Cr-OS-???. 304 305 @param prop_dict: a dictionary we expect the Cros label to be in. 306 @return: A lower case product area, eg: video, factory, ui. 307 """ 308 label = prop_dict.get('label') 309 if label and 'Cr-OS-' in label: 310 return label.split('Cr-OS-')[1] 311 312 313 def get_areas(self): 314 """ 315 Parse issue options and return a list of 'Cr-OS' labels. 316 317 @return: a list of Cr-OS labels from crosbug, eg: ['kernel', 'systems'] 318 """ 319 if apiclient_build is None: 320 logging.error('Missing Api-client import. Cannot get area-labels.') 321 return [] 322 323 try: 324 issue_options_dict = self._get_field('issuesConfig') 325 except ProjectHostingApiException as e: 326 logging.error('Unable to determine area labels: %s', str(e)) 327 return [] 328 329 # Since we can request multiple fields at once we need to 330 # retrieve each one from the field options dictionary, even if we're 331 # really only asking for one field. 332 issue_options = issue_options_dict.get('issuesConfig') 333 if issue_options is None: 334 logging.error('The IssueConfig field does not contain issue ' 335 'configuration as a member anymore; The project ' 336 'hosting api might have changed.') 337 return [] 338 339 return filter(None, [self._get_cros_labels(each) 340 for each in self._get_property_values(issue_options) 341 if isinstance(each, dict)]) 342 343 344 def create_issue(self, request_body): 345 """ 346 Convert the request body into an issue on the frontend tracker. 347 348 @param request_body: A python dictionary with key-value pairs 349 that represent the fields of the issue. 350 eg: { 351 'title': 'bug title', 352 'description': 'bug description', 353 'labels': ['Type-Bug'], 354 'owner': {'name': 'owner@'}, 355 'cc': [{'name': 'cc1'}, {'name': 'cc2'}] 356 } 357 Note the title and descriptions fields of a 358 new bug are not optional, all other fields are. 359 @raises: ProjectHostingApiException, if request execution fails. 360 361 @return: The response body, which will contain the metadata of the 362 issue created, or an error response code and information 363 about a failure. 364 """ 365 issues = self._codesite_service.issues() 366 request = issues.insert(projectId=self._project_name, sendEmail=True, 367 body=request_body) 368 return self._execute_request(request) 369 370 371 def update_issue(self, issue_id, request_body): 372 """ 373 Convert the request body into an update on an issue. 374 375 @param request_body: A python dictionary with key-value pairs 376 that represent the fields of the update. 377 eg: 378 { 379 'content': 'comment to add', 380 'updates': 381 { 382 'labels': ['Type-Bug', 'another label'], 383 'owner': 'owner@', 384 'cc': ['cc1@', cc2@'], 385 } 386 } 387 Note the owner and cc fields need to be email 388 addresses the tracker recognizes. 389 @param issue_id: The id of the issue to update. 390 @raises: ProjectHostingApiException, if request execution fails. 391 392 @return: The response body, which will contain information about the 393 update of said issue, or an error response code and information 394 about a failure. 395 """ 396 issues = self._codesite_service.issues() 397 request = issues.comments().insert(projectId=self._project_name, 398 issueId=issue_id, sendEmail=False, 399 body=request_body) 400 return self._execute_request(request) 401 402 403 def _populate_issue_updates(self, t_issue): 404 """ 405 Populates a tracker issue with updates. 406 407 Any issue is useless without it's updates, since the updates will 408 contain both the summary and the comments. We need at least one of 409 those to successfully dedupe. The Api doesn't allow us to grab all this 410 information in one shot because viewing the comments on an issue 411 requires more authority than just viewing it's title. 412 413 @param t_issue: The basic tracker issue, to populate with updates. 414 @raises: ProjectHostingApiException, if request execution fails. 415 416 @returns: A tracker issue, with it's updates. 417 """ 418 updates = self._list_updates(t_issue['id']) 419 t_issue['updates'] = [update['content'] for update in 420 self._get_property_values(updates) 421 if update.get('content')] 422 return t_issue 423 424 425 def get_tracker_issues_by_text(self, search_text, full_text=True, 426 include_dupes=False, label=None): 427 """ 428 Find all Tracker issues that contain the specified search text. 429 430 @param search_text: Anchor text to use in the search. 431 @param full_text: True if we would like an extensive search through 432 issue comments. If False the search will be restricted 433 to just summaries and titles. 434 @param include_dupes: If True, search over both open issues as well as 435 closed issues whose status is 'Duplicate'. If False, 436 only search over open issues. 437 @param label: A string representing a single label to match. 438 439 @return: A list of issues that contain the search text, or an empty list 440 when we're either unable to list issues or none match the text. 441 """ 442 issue_list = [] 443 try: 444 search_space = 'all' if include_dupes else 'open' 445 feed = self.list_issues(q=search_text, can=search_space, 446 label=label) 447 except ProjectHostingApiException as e: 448 logging.error('Unable to search for issues with marker %s: %s', 449 search_text, e) 450 return issue_list 451 452 for t_issue in self._get_property_values(feed): 453 state = t_issue.get(constants.ISSUE_STATE) 454 status = t_issue.get(constants.ISSUE_STATUS) 455 is_open_or_dup = (state == constants.ISSUE_OPEN or 456 (state == constants.ISSUE_CLOSED 457 and status == constants.ISSUE_DUPLICATE)) 458 # All valid issues will have an issue id we can use to retrieve 459 # more information about it. If we encounter a failure mode that 460 # returns a bad Http response code but doesn't throw an exception 461 # we won't find an issue id in the returned json. 462 if t_issue.get('id') and is_open_or_dup: 463 # TODO(beeps): If this method turns into a performance 464 # bottle neck yield each issue and refactor the reporter. 465 # For now passing all issues allows us to detect when 466 # deduping fails, because multiple issues will match a 467 # given query exactly. 468 try: 469 if full_text: 470 issue = Issue(self._populate_issue_updates(t_issue)) 471 else: 472 issue = BaseIssue(t_issue) 473 except ProjectHostingApiException as e: 474 logging.error('Unable to list the updates of issue %s: %s', 475 t_issue.get('id'), str(e)) 476 else: 477 issue_list.append(issue) 478 return issue_list 479 480 481 def get_tracker_issue_by_id(self, issue_id): 482 """ 483 Returns an issue object given the id. 484 485 @param issue_id: A string representing the issue id. 486 487 @return: An Issue object on success or None on failure. 488 """ 489 try: 490 t_issue = self._get_issue(issue_id) 491 return Issue(self._populate_issue_updates(t_issue)) 492 except ProjectHostingApiException as e: 493 logging.error('Creation of an Issue object for %s fails: %s', 494 issue_id, str(e)) 495 return None 496
