Home | History | Annotate | Download | only in release
      1 #!/usr/bin/env python
      2 # Copyright 2014 the V8 project authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """
      7 Script to check for new clusterfuzz issues since the last rolled v8 revision.
      8 
      9 Returns a json list with test case IDs if any.
     10 
     11 Security considerations: The security key and request data must never be
     12 written to public logs. Public automated callers of this script should
     13 suppress stdout and stderr and only process contents of the results_file.
     14 """
     15 
     16 
     17 import argparse
     18 import httplib
     19 import json
     20 import os
     21 import re
     22 import sys
     23 import urllib
     24 import urllib2
     25 
     26 
     27 # Constants to git repos.
     28 BASE_URL = "https://chromium.googlesource.com"
     29 DEPS_LOG = BASE_URL + "/chromium/src/+log/master/DEPS?format=JSON"
     30 
     31 # Constants for retrieving v8 rolls.
     32 CRREV = "https://cr-rev.appspot.com/_ah/api/crrev/v1/commit/%s"
     33 V8_COMMIT_RE = re.compile(
     34     r"^Update V8 to version \d+\.\d+\.\d+ \(based on ([a-fA-F0-9]+)\)\..*")
     35 
     36 # Constants for the clusterfuzz backend.
     37 HOSTNAME = "backend-dot-cluster-fuzz.appspot.com"
     38 
     39 # Crash patterns.
     40 V8_INTERNAL_RE = re.compile(r"^v8::internal.*")
     41 ANY_RE = re.compile(r".*")
     42 
     43 # List of all api requests.
     44 BUG_SPECS = [
     45   {
     46     "args": {
     47       "job_type": "linux_asan_chrome_v8",
     48       "reproducible": "True",
     49       "open": "True",
     50       "bug_information": "",
     51     },
     52     "crash_state": V8_INTERNAL_RE,
     53   },
     54   {
     55     "args": {
     56       "job_type": "linux_asan_d8",
     57       "reproducible": "True",
     58       "open": "True",
     59       "bug_information": "",
     60     },
     61     "crash_state": ANY_RE,
     62   },
     63   {
     64     "args": {
     65       "job_type": "linux_asan_d8_dbg",
     66       "reproducible": "True",
     67       "open": "True",
     68       "bug_information": "",
     69     },
     70     "crash_state": ANY_RE,
     71   },
     72   {
     73     "args": {
     74       "job_type": "linux_asan_d8_ignition_dbg",
     75       "reproducible": "True",
     76       "open": "True",
     77       "bug_information": "",
     78     },
     79     "crash_state": ANY_RE,
     80   },
     81   {
     82     "args": {
     83       "job_type": "linux_asan_d8_v8_arm_dbg",
     84       "reproducible": "True",
     85       "open": "True",
     86       "bug_information": "",
     87     },
     88     "crash_state": ANY_RE,
     89   },
     90   {
     91     "args": {
     92       "job_type": "linux_asan_d8_ignition_v8_arm_dbg",
     93       "reproducible": "True",
     94       "open": "True",
     95       "bug_information": "",
     96     },
     97     "crash_state": ANY_RE,
     98   },
     99   {
    100     "args": {
    101       "job_type": "linux_asan_d8_v8_arm64_dbg",
    102       "reproducible": "True",
    103       "open": "True",
    104       "bug_information": "",
    105     },
    106     "crash_state": ANY_RE,
    107   },
    108   {
    109     "args": {
    110       "job_type": "linux_asan_d8_v8_mipsel_dbg",
    111       "reproducible": "True",
    112       "open": "True",
    113       "bug_information": "",
    114     },
    115     "crash_state": ANY_RE,
    116   },
    117 ]
    118 
    119 
    120 def GetRequest(url):
    121   url_fh = urllib2.urlopen(url, None, 60)
    122   try:
    123     return url_fh.read()
    124   finally:
    125     url_fh.close()
    126 
    127 
    128 def GetLatestV8InChromium():
    129   """Returns the commit position number of the latest v8 roll in chromium."""
    130 
    131   # Check currently rolled v8 revision.
    132   result = GetRequest(DEPS_LOG)
    133   if not result:
    134     return None
    135 
    136   # Strip security header and load json.
    137   commits = json.loads(result[5:])
    138 
    139   git_revision = None
    140   for commit in commits["log"]:
    141     # Get latest commit that matches the v8 roll pattern. Ignore cherry-picks.
    142     match = re.match(V8_COMMIT_RE, commit["message"])
    143     if match:
    144       git_revision = match.group(1)
    145       break
    146   else:
    147     return None
    148 
    149   # Get commit position number for v8 revision.
    150   result = GetRequest(CRREV % git_revision)
    151   if not result:
    152     return None
    153 
    154   commit = json.loads(result)
    155   assert commit["repo"] == "v8/v8"
    156   return commit["number"]
    157 
    158 
    159 def APIRequest(key, **params):
    160   """Send a request to the clusterfuzz api.
    161 
    162   Returns a json dict of the response.
    163   """
    164 
    165   params["api_key"] = key
    166   params = urllib.urlencode(params)
    167 
    168   headers = {"Content-type": "application/x-www-form-urlencoded"}
    169 
    170   try:
    171     conn = httplib.HTTPSConnection(HOSTNAME)
    172     conn.request("POST", "/_api/", params, headers)
    173 
    174     response = conn.getresponse()
    175 
    176     # Never leak "data" into public logs.
    177     data = response.read()
    178   except:
    179     raise Exception("ERROR: Connection problem.")
    180 
    181   try:
    182     return json.loads(data)
    183   except:
    184     raise Exception("ERROR: Could not read response. Is your key valid?")
    185 
    186   return None
    187 
    188 
    189 def Main():
    190   parser = argparse.ArgumentParser()
    191   parser.add_argument("-k", "--key-file", required=True,
    192                       help="A file with the clusterfuzz api key.")
    193   parser.add_argument("-r", "--results-file",
    194                       help="A file to write the results to.")
    195   options = parser.parse_args()
    196 
    197   # Get api key. The key's content must never be logged.
    198   assert options.key_file
    199   with open(options.key_file) as f:
    200     key = f.read().strip()
    201   assert key
    202 
    203   revision_number = GetLatestV8InChromium()
    204 
    205   results = []
    206   for spec in BUG_SPECS:
    207     args = dict(spec["args"])
    208     # Use incremented revision as we're interested in all revision greater than
    209     # what's currently rolled into chromium.
    210     if revision_number:
    211       args["revision_greater_or_equal"] = str(int(revision_number) + 1)
    212 
    213     # Never print issue details in public logs.
    214     issues = APIRequest(key, **args)
    215     assert issues is not None
    216     for issue in issues:
    217       if (re.match(spec["crash_state"], issue["crash_state"]) and
    218           not issue.get('has_bug_flag')):
    219         results.append(issue["id"])
    220 
    221   if options.results_file:
    222     with open(options.results_file, "w") as f:
    223       f.write(json.dumps(results))
    224   else:
    225     print results
    226 
    227 
    228 if __name__ == "__main__":
    229   sys.exit(Main())
    230