autotest_lib/tko/models.py

import json
import os

from autotest_lib.server.hosts import file_store
from autotest_lib.client.common_lib import utils
from autotest_lib.tko import tast
from autotest_lib.tko import utils as tko_utils


class job(object):
    """Represents a job."""

    def __init__(self, dir, user, label, machine, queued_time, started_time,
                 finished_time, machine_owner, machine_group, aborted_by,
                 aborted_on, keyval_dict):
        self.dir = dir
        self.tests = []
        self.user = user
        self.label = label
        self.machine = machine
        self.queued_time = queued_time
        self.started_time = started_time
        self.finished_time = finished_time
        self.machine_owner = machine_owner
        self.machine_group = machine_group
        self.aborted_by = aborted_by
        self.aborted_on = aborted_on
        self.keyval_dict = keyval_dict
        self.afe_parent_job_id = None
        self.build_version = None
        self.suite = None
        self.board = None
        self.job_idx = None
        # id of the corresponding tko_task_references entry.
        # This table is used to refer to skylab task / afe job corresponding to
        # this tko_job.
        self.task_reference_id = None

    @staticmethod
    def read_keyval(dir):
        """
        Read job keyval files.

        @param dir: String name of directory containing job keyval files.

        @return A dictionary containing job keyvals.

        """
        dir = os.path.normpath(dir)
        top_dir = tko_utils.find_toplevel_job_dir(dir)
        if not top_dir:
            top_dir = dir
        assert(dir.startswith(top_dir))

        # Pull in and merge all the keyval files, with higher-level
        # overriding values in the lower-level ones.
        keyval = {}
        while True:
            try:
                upper_keyval = utils.read_keyval(dir)
                # HACK: exclude hostname from the override - this is a special
                # case where we want lower to override higher.
                if 'hostname' in upper_keyval and 'hostname' in keyval:
                    del upper_keyval['hostname']
                keyval.update(upper_keyval)
            except IOError:
                pass  # If the keyval can't be read just move on to the next.
            if dir == top_dir:
                break
            else:
                assert(dir != '/')
                dir = os.path.dirname(dir)
        return keyval


class kernel(object):
    """Represents a kernel."""

    def __init__(self, base, patches, kernel_hash):
        self.base = base
        self.patches = patches
        self.kernel_hash = kernel_hash


    @staticmethod
    def compute_hash(base, hashes):
        """Compute a hash given the base string and hashes for each patch.

        @param base: A string representing the kernel base.
        @param hashes: A list of hashes, where each hash is associated with a
            patch of this kernel.

        @return A string representing the computed hash.

        """
        key_string = ','.join([base] + hashes)
        return utils.hash('md5', key_string).hexdigest()


class test(object):
    """Represents a test."""

    def __init__(self, subdir, testname, status, reason, test_kernel,
                 machine, started_time, finished_time, iterations,
                 attributes, perf_values, labels):
        self.subdir = subdir
        self.testname = testname
        self.status = status
        self.reason = reason
        self.kernel = test_kernel
        self.machine = machine
        self.started_time = started_time
        self.finished_time = finished_time
        self.iterations = iterations
        self.attributes = attributes
        self.perf_values = perf_values
        self.labels = labels


    @staticmethod
    def load_iterations(keyval_path):
        """Abstract method to load a list of iterations from a keyval file.

        @param keyval_path: String path to a keyval file.

        @return A list of iteration objects.

        """
        raise NotImplementedError


    @staticmethod
    def load_perf_values(perf_values_file):
        """Loads perf values from a perf measurements file.

        @param perf_values_file: The string path to a perf measurements file.

        @return A list of perf_value_iteration objects.

        """
        raise NotImplementedError


    @classmethod
    def parse_test(cls, job, subdir, testname, status, reason, test_kernel,
                   started_time, finished_time, existing_instance=None):
        """
        Parse test result files to construct a complete test instance.

        Given a job and the basic metadata about the test that can be
        extracted from the status logs, parse the test result files (keyval
        files and perf measurement files) and use them to construct a complete
        test instance.

        @param job: A job object.
        @param subdir: The string subdirectory name for the given test.
        @param testname: The name of the test.
        @param status: The status of the test.
        @param reason: The reason string for the test.
        @param test_kernel: The kernel of the test.
        @param started_time: The start time of the test.
        @param finished_time: The finish time of the test.
        @param existing_instance: An existing test instance.

        @return A test instance that has the complete information.

        """
        tko_utils.dprint("parsing test %s %s" % (subdir, testname))

        if tast.is_tast_test(testname):
            attributes, perf_values = tast.load_tast_test_aux_results(job,
                                                                      testname)
            iterations = []
        elif subdir:
            # Grab iterations from the results keyval.
            iteration_keyval = os.path.join(job.dir, subdir,
                                            'results', 'keyval')
            iterations = cls.load_iterations(iteration_keyval)

            # Grab perf values from the perf measurements file.
            perf_values_file = os.path.join(job.dir, subdir,
                                            'results', 'results-chart.json')
            perf_values = {}
            if os.path.exists(perf_values_file):
                with open(perf_values_file, 'r') as fp:
                    contents = fp.read()
                if contents:
                    perf_values = json.loads(contents)

            # Grab test attributes from the subdir keyval.
            test_keyval = os.path.join(job.dir, subdir, 'keyval')
            attributes = test.load_attributes(test_keyval)
        else:
            iterations = []
            perf_values = {}
            attributes = {}

        # Grab test+host attributes from the host keyval.
        host_keyval = cls.parse_host_keyval(job.dir, job.machine)
        attributes.update(dict(('host-%s' % k, v)
                               for k, v in host_keyval.iteritems()))

        if existing_instance:
            def constructor(*args, **dargs):
                """Initializes an existing test instance."""
                existing_instance.__init__(*args, **dargs)
                return existing_instance
        else:
            constructor = cls

        return constructor(subdir, testname, status, reason, test_kernel,
                           job.machine, started_time, finished_time,
                           iterations, attributes, perf_values, [])


    @classmethod
    def parse_partial_test(cls, job, subdir, testname, reason, test_kernel,
                           started_time):
        """
        Create a test instance representing a partial test result.

        Given a job and the basic metadata available when a test is
        started, create a test instance representing the partial result.
        Assume that since the test is not complete there are no results files
        actually available for parsing.

        @param job: A job object.
        @param subdir: The string subdirectory name for the given test.
        @param testname: The name of the test.
        @param reason: The reason string for the test.
        @param test_kernel: The kernel of the test.
        @param started_time: The start time of the test.

        @return A test instance that has partial test information.

        """
        tko_utils.dprint('parsing partial test %s %s' % (subdir, testname))

        return cls(subdir, testname, 'RUNNING', reason, test_kernel,
                   job.machine, started_time, None, [], {}, [], [])


    @staticmethod
    def load_attributes(keyval_path):
        """
        Load test attributes from a test keyval path.

        Load the test attributes into a dictionary from a test
        keyval path. Does not assume that the path actually exists.

        @param keyval_path: The string path to a keyval file.

        @return A dictionary representing the test keyvals.

        """
        if not os.path.exists(keyval_path):
            return {}
        return utils.read_keyval(keyval_path)


    @staticmethod
    def _parse_keyval(job_dir, sub_keyval_path):
        """
        Parse a file of keyvals.

        @param job_dir: The string directory name of the associated job.
        @param sub_keyval_path: Path to a keyval file relative to job_dir.

        @return A dictionary representing the keyvals.

        """
        # The "real" job dir may be higher up in the directory tree.
        job_dir = tko_utils.find_toplevel_job_dir(job_dir)
        if not job_dir:
            return {}  # We can't find a top-level job dir with job keyvals.

        # The keyval is <job_dir>/`sub_keyval_path` if it exists.
        keyval_path = os.path.join(job_dir, sub_keyval_path)
        if os.path.isfile(keyval_path):
            return utils.read_keyval(keyval_path)
        else:
            return {}


    @staticmethod
    def parse_host_keyval(job_dir, hostname):
        """
        Parse host keyvals.

        @param job_dir: The string directory name of the associated job.
        @param hostname: The string hostname.

        @return A dictionary representing the host keyvals.

        """
        keyval_path = os.path.join('host_keyvals', hostname)
        # The host keyval is <job_dir>/host_keyvals/<hostname> if it exists.
        # Otherwise we're running on Skylab which uses hostinfo.
        if not os.path.exists(os.path.join(job_dir, keyval_path)):
            tko_utils.dprint("trying to use hostinfo")
            try:
                return _parse_hostinfo_keyval(job_dir, hostname)
            except Exception as e:
                # If anything goes wrong, log it and just use the old flow.
                tko_utils.dprint("tried using hostinfo: %s" % e)
        return test._parse_keyval(job_dir, keyval_path)


    @staticmethod
    def parse_job_keyval(job_dir):
        """
        Parse job keyvals.

        @param job_dir: The string directory name of the associated job.

        @return A dictionary representing the job keyvals.

        """
        # The job keyval is <job_dir>/keyval if it exists.
        return test._parse_keyval(job_dir, 'keyval')


def _parse_hostinfo_keyval(job_dir, hostname):
    """
    Parse host keyvals from hostinfo.

    @param job_dir: The string directory name of the associated job.
    @param hostname: The string hostname.

    @return A dictionary representing the host keyvals.

    """
    # The hostinfo path looks like:
    # host_info_store/chromeos6-row4-rack11-host6.store
    #
    # TODO(ayatane): We should pass hostinfo path explicitly.
    subdir = 'host_info_store'
    hostinfo_path = os.path.join(job_dir, subdir, hostname + '.store')
    store = file_store.FileStore(hostinfo_path)
    hostinfo = store.get()
    # TODO(ayatane): Investigate if urllib.quote is better.
    label_string = ','.join(label.replace(':', '%3A')
                            for label in hostinfo.labels)
    return {'labels': label_string, 'platform': hostinfo.model}


class patch(object):
    """Represents a patch."""

    def __init__(self, spec, reference, hash):
        self.spec = spec
        self.reference = reference
        self.hash = hash


class iteration(object):
    """Represents an iteration."""

    def __init__(self, index, attr_keyval, perf_keyval):
        self.index = index
        self.attr_keyval = attr_keyval
        self.perf_keyval = perf_keyval


    @staticmethod
    def parse_line_into_dicts(line, attr_dict, perf_dict):
        """
        Abstract method to parse a keyval line and insert it into a dictionary.

        @param line: The string line to parse.
        @param attr_dict: Dictionary of generic iteration attributes.
        @param perf_dict: Dictionary of iteration performance results.

        """
        raise NotImplementedError


    @classmethod
    def load_from_keyval(cls, keyval_path):
        """
        Load a list of iterations from an iteration keyval file.

        Keyval data from separate iterations is separated by blank
        lines. Makes use of the parse_line_into_dicts method to
        actually parse the individual lines.

        @param keyval_path: The string path to a keyval file.

        @return A list of iteration objects.

        """
        if not os.path.exists(keyval_path):
            return []

        iterations = []
        index = 1
        attr, perf = {}, {}
        for line in file(keyval_path):
            line = line.strip()
            if line:
                cls.parse_line_into_dicts(line, attr, perf)
            else:
                iterations.append(cls(index, attr, perf))
                index += 1
                attr, perf = {}, {}
        if attr or perf:
            iterations.append(cls(index, attr, perf))
        return iterations


class perf_value_iteration(object):
    """Represents a perf value iteration."""

    def __init__(self, index, perf_measurements):
        """
        Initializes the perf values for a particular test iteration.

        @param index: The integer iteration number.
        @param perf_measurements: A list of dictionaries, where each dictionary
            contains the information for a measured perf metric from the
            current iteration.

        """
        self.index = index
        self.perf_measurements = perf_measurements


    def add_measurement(self, measurement):
        """
        Appends to the list of perf measurements for this iteration.

        @param measurement: A dictionary containing information for a measured
            perf metric.

        """
        self.perf_measurements.append(measurement)


    @staticmethod
    def parse_line_into_dict(line):
        """
        Abstract method to parse an individual perf measurement line.

        @param line: A string line from the perf measurement output file.

        @return A dicionary representing the information for a measured perf
            metric from one line of the perf measurement output file, or an
            empty dictionary if the line cannot be parsed successfully.

        """
        raise NotImplementedError