Home | History | Annotate | Download | only in parsers 
      1 import re, os
      2 from autotest_lib.tko import utils as tko_utils, models, status_lib
      3 from autotest_lib.tko.parsers import base
      4 
      5 class NoHostnameError(Exception):
      6     pass
      7 
      8 
      9 class BoardLabelError(Exception):
     10     pass
     11 
     12 
     13 class job(models.job):
     14     def __init__(self, dir):
     15         job_dict = job.load_from_dir(dir)
     16         super(job, self).__init__(dir, **job_dict)
     17 
     18 
     19     @classmethod
     20     def load_from_dir(cls, dir):
     21         keyval = cls.read_keyval(dir)
     22         tko_utils.dprint(str(keyval))
     23 
     24         user = keyval.get("user", None)
     25         label = keyval.get("label", None)
     26         queued_time = tko_utils.get_timestamp(keyval, "job_queued")
     27         started_time = tko_utils.get_timestamp(keyval, "job_started")
     28         finished_time = tko_utils.get_timestamp(keyval, "job_finished")
     29         machine = cls.determine_hostname(keyval, dir)
     30         machine_group = cls.determine_machine_group(machine, dir)
     31         machine_owner = keyval.get("owner", None)
     32 
     33         aborted_by = keyval.get("aborted_by", None)
     34         aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
     35 
     36         return {"user": user, "label": label, "machine": machine,
     37                 "queued_time": queued_time, "started_time": started_time,
     38                 "finished_time": finished_time, "machine_owner": machine_owner,
     39                 "machine_group": machine_group, "aborted_by": aborted_by,
     40                 "aborted_on": aborted_at, "keyval_dict": keyval}
     41 
     42 
     43     @classmethod
     44     def determine_hostname(cls, keyval, job_dir):
     45         host_group_name = keyval.get("host_group_name", None)
     46         machine = keyval.get("hostname", "")
     47         is_multimachine = "," in machine
     48 
     49         # determine what hostname to use
     50         if host_group_name:
     51             if is_multimachine or not machine:
     52                 tko_utils.dprint("Using host_group_name %r instead of "
     53                                  "machine name." % host_group_name)
     54                 machine = host_group_name
     55         elif is_multimachine:
     56             try:
     57                 machine = job.find_hostname(job_dir) # find a unique hostname
     58             except NoHostnameError:
     59                 pass  # just use the comma-separated name
     60 
     61         tko_utils.dprint("MACHINE NAME: %s" % machine)
     62         return machine
     63 
     64 
     65     @classmethod
     66     def determine_machine_group(cls, hostname, job_dir):
     67         machine_groups = set()
     68         for individual_hostname in hostname.split(","):
     69             host_keyval = models.test.parse_host_keyval(job_dir,
     70                                                         individual_hostname)
     71             if not host_keyval:
     72                 tko_utils.dprint('Unable to parse host keyval for %s'
     73                                  % individual_hostname)
     74             elif 'labels' in host_keyval:
     75                 # Use board label as machine group. This is to avoid the
     76                 # confusion of multiple boards mapping to the same platform in
     77                 # wmatrix. With this change, wmatrix will group tests with the
     78                 # same board, rather than the same platform.
     79                 labels = host_keyval['labels'].split(',')
     80                 board_labels = [l[8:] for l in labels
     81                                if l.startswith('board%3A')]
     82                 if board_labels:
     83                     # Testbeds have multiple boards so concat them into a
     84                     # single string then add it to the machine_groups list.
     85                     machine_groups.add(','.join(board_labels))
     86                 else:
     87                     error = ('Failed to retrieve board label from host labels: '
     88                              '%s' % host_keyval['labels'])
     89                     tko_utils.dprint(error)
     90                     raise BoardLabelError(error)
     91             elif "platform" in host_keyval:
     92                 machine_groups.add(host_keyval["platform"])
     93         machine_group = ",".join(sorted(machine_groups))
     94         tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
     95         return machine_group
     96 
     97 
     98     @staticmethod
     99     def find_hostname(path):
    100         hostname = os.path.join(path, "sysinfo", "hostname")
    101         try:
    102             machine = open(hostname).readline().rstrip()
    103             return machine
    104         except Exception:
    105             tko_utils.dprint("Could not read a hostname from "
    106                              "sysinfo/hostname")
    107 
    108         uname = os.path.join(path, "sysinfo", "uname_-a")
    109         try:
    110             machine = open(uname).readline().split()[1]
    111             return machine
    112         except Exception:
    113             tko_utils.dprint("Could not read a hostname from "
    114                              "sysinfo/uname_-a")
    115 
    116         raise NoHostnameError("Unable to find a machine name")
    117 
    118 
    119 class kernel(models.kernel):
    120     def __init__(self, job, verify_ident=None):
    121         kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
    122         super(kernel, self).__init__(**kernel_dict)
    123 
    124 
    125     @staticmethod
    126     def load_from_dir(dir, verify_ident=None):
    127         # try and load the booted kernel version
    128         attributes = False
    129         i = 1
    130         build_dir = os.path.join(dir, "build")
    131         while True:
    132             if not os.path.exists(build_dir):
    133                 break
    134             build_log = os.path.join(build_dir, "debug", "build_log")
    135             attributes = kernel.load_from_build_log(build_log)
    136             if attributes:
    137                 break
    138             i += 1
    139             build_dir = os.path.join(dir, "build.%d" % (i))
    140 
    141         if not attributes:
    142             if verify_ident:
    143                 base = verify_ident
    144             else:
    145                 base = kernel.load_from_sysinfo(dir)
    146             patches = []
    147             hashes = []
    148         else:
    149             base, patches, hashes = attributes
    150         tko_utils.dprint("kernel.__init__() found kernel version %s"
    151                          % base)
    152 
    153         # compute the kernel hash
    154         if base == "UNKNOWN":
    155             kernel_hash = "UNKNOWN"
    156         else:
    157             kernel_hash = kernel.compute_hash(base, hashes)
    158 
    159         return {"base": base, "patches": patches,
    160                 "kernel_hash": kernel_hash}
    161 
    162 
    163     @staticmethod
    164     def load_from_sysinfo(path):
    165         for subdir in ("reboot1", ""):
    166             uname_path = os.path.join(path, "sysinfo", subdir,
    167                                       "uname_-a")
    168             if not os.path.exists(uname_path):
    169                 continue
    170             uname = open(uname_path).readline().split()
    171             return re.sub("-autotest$", "", uname[2])
    172         return "UNKNOWN"
    173 
    174 
    175     @staticmethod
    176     def load_from_build_log(path):
    177         if not os.path.exists(path):
    178             return None
    179 
    180         base, patches, hashes = "UNKNOWN", [], []
    181         for line in file(path):
    182             head, rest = line.split(": ", 1)
    183             rest = rest.split()
    184             if head == "BASE":
    185                 base = rest[0]
    186             elif head == "PATCH":
    187                 patches.append(patch(*rest))
    188                 hashes.append(rest[2])
    189         return base, patches, hashes
    190 
    191 
    192 class test(models.test):
    193     def __init__(self, subdir, testname, status, reason, test_kernel,
    194                  machine, started_time, finished_time, iterations,
    195                  attributes, labels):
    196         # for backwards compatibility with the original parser
    197         # implementation, if there is no test version we need a NULL
    198         # value to be used; also, if there is a version it should
    199         # be terminated by a newline
    200         if "version" in attributes:
    201             attributes["version"] = str(attributes["version"])
    202         else:
    203             attributes["version"] = None
    204 
    205         super(test, self).__init__(subdir, testname, status, reason,
    206                                    test_kernel, machine, started_time,
    207                                    finished_time, iterations,
    208                                    attributes, labels)
    209 
    210 
    211     @staticmethod
    212     def load_iterations(keyval_path):
    213         return iteration.load_from_keyval(keyval_path)
    214 
    215 
    216 class patch(models.patch):
    217     def __init__(self, spec, reference, hash):
    218         tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
    219         super(patch, self).__init__(spec, reference, hash)
    220         self.spec = spec
    221         self.reference = reference
    222         self.hash = hash
    223 
    224 
    225 class iteration(models.iteration):
    226     @staticmethod
    227     def parse_line_into_dicts(line, attr_dict, perf_dict):
    228         key, value = line.split("=", 1)
    229         perf_dict[key] = value
    230 
    231 
    232 class status_line(object):
    233     def __init__(self, indent, status, subdir, testname, reason,
    234                  optional_fields):
    235         # pull out the type & status of the line
    236         if status == "START":
    237             self.type = "START"
    238             self.status = None
    239         elif status.startswith("END "):
    240             self.type = "END"
    241             self.status = status[4:]
    242         else:
    243             self.type = "STATUS"
    244             self.status = status
    245         assert (self.status is None or
    246                 self.status in status_lib.statuses)
    247 
    248         # save all the other parameters
    249         self.indent = indent
    250         self.subdir = self.parse_name(subdir)
    251         self.testname = self.parse_name(testname)
    252         self.reason = reason
    253         self.optional_fields = optional_fields
    254 
    255 
    256     @staticmethod
    257     def parse_name(name):
    258         if name == "----":
    259             return None
    260         return name
    261 
    262 
    263     @staticmethod
    264     def is_status_line(line):
    265         return re.search(r"^\t*(\S[^\t]*\t){3}", line) is not None
    266 
    267 
    268     @classmethod
    269     def parse_line(cls, line):
    270         if not status_line.is_status_line(line):
    271             return None
    272         match = re.search(r"^(\t*)(.*)$", line, flags=re.DOTALL)
    273         if not match:
    274             # A more useful error message than:
    275             #  AttributeError: 'NoneType' object has no attribute 'groups'
    276             # to help us debug WTF happens on occasion here.
    277             raise RuntimeError("line %r could not be parsed." % line)
    278         indent, line = match.groups()
    279         indent = len(indent)
    280 
    281         # split the line into the fixed and optional fields
    282         parts = line.rstrip("\n").split("\t")
    283 
    284         part_index = 3
    285         status, subdir, testname = parts[0:part_index]
    286 
    287         # all optional parts should be of the form "key=value". once we've found
    288         # a non-matching part, treat it and the rest of the parts as the reason.
    289         optional_fields = {}
    290         while part_index < len(parts):
    291             kv = re.search(r"^(\w+)=(.+)", parts[part_index])
    292             if not kv:
    293                 break
    294 
    295             optional_fields[kv.group(1)] = kv.group(2)
    296             part_index += 1
    297 
    298         reason = "\t".join(parts[part_index:])
    299 
    300         # build up a new status_line and return it
    301         return cls(indent, status, subdir, testname, reason,
    302                    optional_fields)
    303 
    304 
    305 class parser(base.parser):
    306     @staticmethod
    307     def make_job(dir):
    308         return job(dir)
    309 
    310 
    311     def state_iterator(self, buffer):
    312         new_tests = []
    313         boot_count = 0
    314         group_subdir = None
    315         sought_level = 0
    316         stack = status_lib.status_stack()
    317         current_kernel = kernel(self.job)
    318         boot_in_progress = False
    319         alert_pending = None
    320         started_time = None
    321 
    322         while not self.finished or buffer.size():
    323             # stop processing once the buffer is empty
    324             if buffer.size() == 0:
    325                 yield new_tests
    326                 new_tests = []
    327                 continue
    328 
    329             # parse the next line
    330             line = buffer.get()
    331             tko_utils.dprint('\nSTATUS: ' + line.strip())
    332             line = status_line.parse_line(line)
    333             if line is None:
    334                 tko_utils.dprint('non-status line, ignoring')
    335                 continue # ignore non-status lines
    336 
    337             # have we hit the job start line?
    338             if (line.type == "START" and not line.subdir and
    339                 not line.testname):
    340                 sought_level = 1
    341                 tko_utils.dprint("found job level start "
    342                                  "marker, looking for level "
    343                                  "1 groups now")
    344                 continue
    345 
    346             # have we hit the job end line?
    347             if (line.type == "END" and not line.subdir and
    348                 not line.testname):
    349                 tko_utils.dprint("found job level end "
    350                                  "marker, looking for level "
    351                                  "0 lines now")
    352                 sought_level = 0
    353 
    354             # START line, just push another layer on to the stack
    355             # and grab the start time if this is at the job level
    356             # we're currently seeking
    357             if line.type == "START":
    358                 group_subdir = None
    359                 stack.start()
    360                 if line.indent == sought_level:
    361                     started_time = \
    362                                  tko_utils.get_timestamp(
    363                         line.optional_fields, "timestamp")
    364                 tko_utils.dprint("start line, ignoring")
    365                 continue
    366             # otherwise, update the status on the stack
    367             else:
    368                 tko_utils.dprint("GROPE_STATUS: %s" %
    369                                  [stack.current_status(),
    370                                   line.status, line.subdir,
    371                                   line.testname, line.reason])
    372                 stack.update(line.status)
    373 
    374             if line.status == "ALERT":
    375                 tko_utils.dprint("job level alert, recording")
    376                 alert_pending = line.reason
    377                 continue
    378 
    379             # ignore Autotest.install => GOOD lines
    380             if (line.testname == "Autotest.install" and
    381                 line.status == "GOOD"):
    382                 tko_utils.dprint("Successful Autotest "
    383                                  "install, ignoring")
    384                 continue
    385 
    386             # ignore END lines for a reboot group
    387             if (line.testname == "reboot" and line.type == "END"):
    388                 tko_utils.dprint("reboot group, ignoring")
    389                 continue
    390 
    391             # convert job-level ABORTs into a 'CLIENT_JOB' test, and
    392             # ignore other job-level events
    393             if line.testname is None:
    394                 if (line.status == "ABORT" and
    395                     line.type != "END"):
    396                     line.testname = "CLIENT_JOB"
    397                 else:
    398                     tko_utils.dprint("job level event, "
    399                                     "ignoring")
    400                     continue
    401 
    402             # use the group subdir for END lines
    403             if line.type == "END":
    404                 line.subdir = group_subdir
    405 
    406             # are we inside a block group?
    407             if (line.indent != sought_level and
    408                 line.status != "ABORT" and
    409                 not line.testname.startswith('reboot.')):
    410                 if line.subdir:
    411                     tko_utils.dprint("set group_subdir: "
    412                                      + line.subdir)
    413                     group_subdir = line.subdir
    414                 tko_utils.dprint("ignoring incorrect indent "
    415                                  "level %d != %d," %
    416                                  (line.indent, sought_level))
    417                 continue
    418 
    419             # use the subdir as the testname, except for
    420             # boot.* and kernel.* tests
    421             if (line.testname is None or
    422                 not re.search(r"^(boot(\.\d+)?$|kernel\.)",
    423                               line.testname)):
    424                 if line.subdir and '.' in line.subdir:
    425                     line.testname = line.subdir
    426 
    427             # has a reboot started?
    428             if line.testname == "reboot.start":
    429                 started_time = tko_utils.get_timestamp(
    430                     line.optional_fields, "timestamp")
    431                 tko_utils.dprint("reboot start event, "
    432                                  "ignoring")
    433                 boot_in_progress = True
    434                 continue
    435 
    436             # has a reboot finished?
    437             if line.testname == "reboot.verify":
    438                 line.testname = "boot.%d" % boot_count
    439                 tko_utils.dprint("reboot verified")
    440                 boot_in_progress = False
    441                 verify_ident = line.reason.strip()
    442                 current_kernel = kernel(self.job, verify_ident)
    443                 boot_count += 1
    444 
    445             if alert_pending:
    446                 line.status = "ALERT"
    447                 line.reason = alert_pending
    448                 alert_pending = None
    449 
    450             # create the actual test object
    451             finished_time = tko_utils.get_timestamp(
    452                 line.optional_fields, "timestamp")
    453             final_status = stack.end()
    454             tko_utils.dprint("Adding: "
    455                              "%s\nSubdir:%s\nTestname:%s\n%s" %
    456                              (final_status, line.subdir,
    457                               line.testname, line.reason))
    458             new_test = test.parse_test(self.job, line.subdir,
    459                                        line.testname,
    460                                        final_status, line.reason,
    461                                        current_kernel,
    462                                        started_time,
    463                                        finished_time)
    464             started_time = None
    465             new_tests.append(new_test)
    466 
    467         # the job is finished, but we never came back from reboot
    468         if boot_in_progress:
    469             testname = "boot.%d" % boot_count
    470             reason = "machine did not return from reboot"
    471             tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
    472                               "Testname:%s\n%s")
    473                              % (testname, reason))
    474             new_test = test.parse_test(self.job, None, testname,
    475                                        "ABORT", reason,
    476                                        current_kernel, None, None)
    477             new_tests.append(new_test)
    478         yield new_tests
    479