Home | History | Annotate | Download | only in parsers 
      1 import re, os
      2 
      3 from autotest_lib.client.common_lib import utils as common_utils
      4 from autotest_lib.tko import utils as tko_utils, models, status_lib
      5 from autotest_lib.tko.parsers import base
      6 
      7 
      8 class NoHostnameError(Exception):
      9     pass
     10 
     11 
     12 class BoardLabelError(Exception):
     13     pass
     14 
     15 
     16 class job(models.job):
     17     def __init__(self, dir):
     18         job_dict = job.load_from_dir(dir)
     19         super(job, self).__init__(dir, **job_dict)
     20 
     21 
     22     @classmethod
     23     def load_from_dir(cls, dir):
     24         keyval = cls.read_keyval(dir)
     25         tko_utils.dprint(str(keyval))
     26 
     27         user = keyval.get("user", None)
     28         label = keyval.get("label", None)
     29         queued_time = tko_utils.get_timestamp(keyval, "job_queued")
     30         started_time = tko_utils.get_timestamp(keyval, "job_started")
     31         finished_time = tko_utils.get_timestamp(keyval, "job_finished")
     32         machine = cls.determine_hostname(keyval, dir)
     33         machine_group = cls.determine_machine_group(machine, dir)
     34         machine_owner = keyval.get("owner", None)
     35 
     36         aborted_by = keyval.get("aborted_by", None)
     37         aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
     38 
     39         return {"user": user, "label": label, "machine": machine,
     40                 "queued_time": queued_time, "started_time": started_time,
     41                 "finished_time": finished_time, "machine_owner": machine_owner,
     42                 "machine_group": machine_group, "aborted_by": aborted_by,
     43                 "aborted_on": aborted_at, "keyval_dict": keyval}
     44 
     45 
     46     @classmethod
     47     def determine_hostname(cls, keyval, job_dir):
     48         host_group_name = keyval.get("host_group_name", None)
     49         machine = keyval.get("hostname", "")
     50         is_multimachine = "," in machine
     51 
     52         # determine what hostname to use
     53         if host_group_name:
     54             if is_multimachine or not machine:
     55                 tko_utils.dprint("Using host_group_name %r instead of "
     56                                  "machine name." % host_group_name)
     57                 machine = host_group_name
     58         elif is_multimachine:
     59             try:
     60                 machine = job.find_hostname(job_dir) # find a unique hostname
     61             except NoHostnameError:
     62                 pass  # just use the comma-separated name
     63 
     64         tko_utils.dprint("MACHINE NAME: %s" % machine)
     65         return machine
     66 
     67 
     68     @classmethod
     69     def determine_machine_group(cls, hostname, job_dir):
     70         machine_groups = set()
     71         for individual_hostname in hostname.split(","):
     72             host_keyval = models.test.parse_host_keyval(job_dir,
     73                                                         individual_hostname)
     74             if not host_keyval:
     75                 tko_utils.dprint('Unable to parse host keyval for %s'
     76                                  % individual_hostname)
     77             elif 'labels' in host_keyval:
     78                 # Use board label as machine group. This is to avoid the
     79                 # confusion of multiple boards mapping to the same platform in
     80                 # wmatrix. With this change, wmatrix will group tests with the
     81                 # same board, rather than the same platform.
     82                 labels = host_keyval['labels'].split(',')
     83                 board_labels = [l[8:] for l in labels
     84                                if l.startswith('board%3A')]
     85                 if board_labels and len(board_labels) == 1:
     86                     machine_groups.add(board_labels[0])
     87                 else:
     88                     error = ('Failed to retrieve board label from host labels: '
     89                              '%s' % host_keyval['labels'])
     90                     tko_utils.dprint(error)
     91                     raise BoardLabelError(error)
     92             elif "platform" in host_keyval:
     93                 machine_groups.add(host_keyval["platform"])
     94         machine_group = ",".join(sorted(machine_groups))
     95         tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
     96         return machine_group
     97 
     98 
     99     @staticmethod
    100     def find_hostname(path):
    101         hostname = os.path.join(path, "sysinfo", "hostname")
    102         try:
    103             machine = open(hostname).readline().rstrip()
    104             return machine
    105         except Exception:
    106             tko_utils.dprint("Could not read a hostname from "
    107                              "sysinfo/hostname")
    108 
    109         uname = os.path.join(path, "sysinfo", "uname_-a")
    110         try:
    111             machine = open(uname).readline().split()[1]
    112             return machine
    113         except Exception:
    114             tko_utils.dprint("Could not read a hostname from "
    115                              "sysinfo/uname_-a")
    116 
    117         raise NoHostnameError("Unable to find a machine name")
    118 
    119 
    120 class kernel(models.kernel):
    121     def __init__(self, job, verify_ident=None):
    122         kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
    123         super(kernel, self).__init__(**kernel_dict)
    124 
    125 
    126     @staticmethod
    127     def load_from_dir(dir, verify_ident=None):
    128         # try and load the booted kernel version
    129         attributes = False
    130         i = 1
    131         build_dir = os.path.join(dir, "build")
    132         while True:
    133             if not os.path.exists(build_dir):
    134                 break
    135             build_log = os.path.join(build_dir, "debug", "build_log")
    136             attributes = kernel.load_from_build_log(build_log)
    137             if attributes:
    138                 break
    139             i += 1
    140             build_dir = os.path.join(dir, "build.%d" % (i))
    141 
    142         if not attributes:
    143             if verify_ident:
    144                 base = verify_ident
    145             else:
    146                 base = kernel.load_from_sysinfo(dir)
    147             patches = []
    148             hashes = []
    149         else:
    150             base, patches, hashes = attributes
    151         tko_utils.dprint("kernel.__init__() found kernel version %s"
    152                          % base)
    153 
    154         # compute the kernel hash
    155         if base == "UNKNOWN":
    156             kernel_hash = "UNKNOWN"
    157         else:
    158             kernel_hash = kernel.compute_hash(base, hashes)
    159 
    160         return {"base": base, "patches": patches,
    161                 "kernel_hash": kernel_hash}
    162 
    163 
    164     @staticmethod
    165     def load_from_sysinfo(path):
    166         for subdir in ("reboot1", ""):
    167             uname_path = os.path.join(path, "sysinfo", subdir,
    168                                       "uname_-a")
    169             if not os.path.exists(uname_path):
    170                 continue
    171             uname = open(uname_path).readline().split()
    172             return re.sub("-autotest$", "", uname[2])
    173         return "UNKNOWN"
    174 
    175 
    176     @staticmethod
    177     def load_from_build_log(path):
    178         if not os.path.exists(path):
    179             return None
    180 
    181         base, patches, hashes = "UNKNOWN", [], []
    182         for line in file(path):
    183             head, rest = line.split(": ", 1)
    184             rest = rest.split()
    185             if head == "BASE":
    186                 base = rest[0]
    187             elif head == "PATCH":
    188                 patches.append(patch(*rest))
    189                 hashes.append(rest[2])
    190         return base, patches, hashes
    191 
    192 
    193 class test(models.test):
    194     def __init__(self, subdir, testname, status, reason, test_kernel,
    195                  machine, started_time, finished_time, iterations,
    196                  attributes, labels):
    197         # for backwards compatibility with the original parser
    198         # implementation, if there is no test version we need a NULL
    199         # value to be used; also, if there is a version it should
    200         # be terminated by a newline
    201         if "version" in attributes:
    202             attributes["version"] = str(attributes["version"])
    203         else:
    204             attributes["version"] = None
    205 
    206         super(test, self).__init__(subdir, testname, status, reason,
    207                                    test_kernel, machine, started_time,
    208                                    finished_time, iterations,
    209                                    attributes, labels)
    210 
    211 
    212     @staticmethod
    213     def load_iterations(keyval_path):
    214         return iteration.load_from_keyval(keyval_path)
    215 
    216 
    217 class patch(models.patch):
    218     def __init__(self, spec, reference, hash):
    219         tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
    220         super(patch, self).__init__(spec, reference, hash)
    221         self.spec = spec
    222         self.reference = reference
    223         self.hash = hash
    224 
    225 
    226 class iteration(models.iteration):
    227     @staticmethod
    228     def parse_line_into_dicts(line, attr_dict, perf_dict):
    229         key, value = line.split("=", 1)
    230         perf_dict[key] = value
    231 
    232 
    233 class status_line(object):
    234     def __init__(self, indent, status, subdir, testname, reason,
    235                  optional_fields):
    236         # pull out the type & status of the line
    237         if status == "START":
    238             self.type = "START"
    239             self.status = None
    240         elif status.startswith("END "):
    241             self.type = "END"
    242             self.status = status[4:]
    243         else:
    244             self.type = "STATUS"
    245             self.status = status
    246         assert (self.status is None or
    247                 self.status in status_lib.statuses)
    248 
    249         # save all the other parameters
    250         self.indent = indent
    251         self.subdir = self.parse_name(subdir)
    252         self.testname = self.parse_name(testname)
    253         self.reason = reason
    254         self.optional_fields = optional_fields
    255 
    256 
    257     @staticmethod
    258     def parse_name(name):
    259         if name == "----":
    260             return None
    261         return name
    262 
    263 
    264     @staticmethod
    265     def is_status_line(line):
    266         return re.search(r"^\t*(\S[^\t]*\t){3}", line) is not None
    267 
    268 
    269     @classmethod
    270     def parse_line(cls, line):
    271         if not status_line.is_status_line(line):
    272             return None
    273         match = re.search(r"^(\t*)(.*)$", line, flags=re.DOTALL)
    274         if not match:
    275             # A more useful error message than:
    276             #  AttributeError: 'NoneType' object has no attribute 'groups'
    277             # to help us debug WTF happens on occasion here.
    278             raise RuntimeError("line %r could not be parsed." % line)
    279         indent, line = match.groups()
    280         indent = len(indent)
    281 
    282         # split the line into the fixed and optional fields
    283         parts = line.rstrip("\n").split("\t")
    284 
    285         part_index = 3
    286         status, subdir, testname = parts[0:part_index]
    287 
    288         # all optional parts should be of the form "key=value". once we've found
    289         # a non-matching part, treat it and the rest of the parts as the reason.
    290         optional_fields = {}
    291         while part_index < len(parts):
    292             kv = re.search(r"^(\w+)=(.+)", parts[part_index])
    293             if not kv:
    294                 break
    295 
    296             optional_fields[kv.group(1)] = kv.group(2)
    297             part_index += 1
    298 
    299         reason = "\t".join(parts[part_index:])
    300 
    301         # build up a new status_line and return it
    302         return cls(indent, status, subdir, testname, reason,
    303                    optional_fields)
    304 
    305 
    306 class parser(base.parser):
    307     @staticmethod
    308     def make_job(dir):
    309         return job(dir)
    310 
    311 
    312     def state_iterator(self, buffer):
    313         new_tests = []
    314         boot_count = 0
    315         group_subdir = None
    316         sought_level = 0
    317         stack = status_lib.status_stack()
    318         current_kernel = kernel(self.job)
    319         boot_in_progress = False
    320         alert_pending = None
    321         started_time = None
    322 
    323         while not self.finished or buffer.size():
    324             # stop processing once the buffer is empty
    325             if buffer.size() == 0:
    326                 yield new_tests
    327                 new_tests = []
    328                 continue
    329 
    330             # parse the next line
    331             line = buffer.get()
    332             tko_utils.dprint('\nSTATUS: ' + line.strip())
    333             line = status_line.parse_line(line)
    334             if line is None:
    335                 tko_utils.dprint('non-status line, ignoring')
    336                 continue # ignore non-status lines
    337 
    338             # have we hit the job start line?
    339             if (line.type == "START" and not line.subdir and
    340                 not line.testname):
    341                 sought_level = 1
    342                 tko_utils.dprint("found job level start "
    343                                  "marker, looking for level "
    344                                  "1 groups now")
    345                 continue
    346 
    347             # have we hit the job end line?
    348             if (line.type == "END" and not line.subdir and
    349                 not line.testname):
    350                 tko_utils.dprint("found job level end "
    351                                  "marker, looking for level "
    352                                  "0 lines now")
    353                 sought_level = 0
    354 
    355             # START line, just push another layer on to the stack
    356             # and grab the start time if this is at the job level
    357             # we're currently seeking
    358             if line.type == "START":
    359                 group_subdir = None
    360                 stack.start()
    361                 if line.indent == sought_level:
    362                     started_time = \
    363                                  tko_utils.get_timestamp(
    364                         line.optional_fields, "timestamp")
    365                 tko_utils.dprint("start line, ignoring")
    366                 continue
    367             # otherwise, update the status on the stack
    368             else:
    369                 tko_utils.dprint("GROPE_STATUS: %s" %
    370                                  [stack.current_status(),
    371                                   line.status, line.subdir,
    372                                   line.testname, line.reason])
    373                 stack.update(line.status)
    374 
    375             if line.status == "ALERT":
    376                 tko_utils.dprint("job level alert, recording")
    377                 alert_pending = line.reason
    378                 continue
    379 
    380             # ignore Autotest.install => GOOD lines
    381             if (line.testname == "Autotest.install" and
    382                 line.status == "GOOD"):
    383                 tko_utils.dprint("Successful Autotest "
    384                                  "install, ignoring")
    385                 continue
    386 
    387             # ignore END lines for a reboot group
    388             if (line.testname == "reboot" and line.type == "END"):
    389                 tko_utils.dprint("reboot group, ignoring")
    390                 continue
    391 
    392             # convert job-level ABORTs into a 'CLIENT_JOB' test, and
    393             # ignore other job-level events
    394             if line.testname is None:
    395                 if (line.status == "ABORT" and
    396                     line.type != "END"):
    397                     line.testname = "CLIENT_JOB"
    398                 else:
    399                     tko_utils.dprint("job level event, "
    400                                     "ignoring")
    401                     continue
    402 
    403             # use the group subdir for END lines
    404             if line.type == "END":
    405                 line.subdir = group_subdir
    406 
    407             # are we inside a block group?
    408             if (line.indent != sought_level and
    409                 line.status != "ABORT" and
    410                 not line.testname.startswith('reboot.')):
    411                 if line.subdir:
    412                     tko_utils.dprint("set group_subdir: "
    413                                      + line.subdir)
    414                     group_subdir = line.subdir
    415                 tko_utils.dprint("ignoring incorrect indent "
    416                                  "level %d != %d," %
    417                                  (line.indent, sought_level))
    418                 continue
    419 
    420             # use the subdir as the testname, except for
    421             # boot.* and kernel.* tests
    422             if (line.testname is None or
    423                 not re.search(r"^(boot(\.\d+)?$|kernel\.)",
    424                               line.testname)):
    425                 if line.subdir and '.' in line.subdir:
    426                     line.testname = line.subdir
    427 
    428             # has a reboot started?
    429             if line.testname == "reboot.start":
    430                 started_time = tko_utils.get_timestamp(
    431                     line.optional_fields, "timestamp")
    432                 tko_utils.dprint("reboot start event, "
    433                                  "ignoring")
    434                 boot_in_progress = True
    435                 continue
    436 
    437             # has a reboot finished?
    438             if line.testname == "reboot.verify":
    439                 line.testname = "boot.%d" % boot_count
    440                 tko_utils.dprint("reboot verified")
    441                 boot_in_progress = False
    442                 verify_ident = line.reason.strip()
    443                 current_kernel = kernel(self.job, verify_ident)
    444                 boot_count += 1
    445 
    446             if alert_pending:
    447                 line.status = "ALERT"
    448                 line.reason = alert_pending
    449                 alert_pending = None
    450 
    451             # create the actual test object
    452             finished_time = tko_utils.get_timestamp(
    453                 line.optional_fields, "timestamp")
    454             final_status = stack.end()
    455             tko_utils.dprint("Adding: "
    456                              "%s\nSubdir:%s\nTestname:%s\n%s" %
    457                              (final_status, line.subdir,
    458                               line.testname, line.reason))
    459             new_test = test.parse_test(self.job, line.subdir,
    460                                        line.testname,
    461                                        final_status, line.reason,
    462                                        current_kernel,
    463                                        started_time,
    464                                        finished_time)
    465             started_time = None
    466             new_tests.append(new_test)
    467 
    468         # the job is finished, but we never came back from reboot
    469         if boot_in_progress:
    470             testname = "boot.%d" % boot_count
    471             reason = "machine did not return from reboot"
    472             tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
    473                               "Testname:%s\n%s")
    474                              % (testname, reason))
    475             new_test = test.parse_test(self.job, None, testname,
    476                                        "ABORT", reason,
    477                                        current_kernel, None, None)
    478             new_tests.append(new_test)
    479         yield new_tests
    480