xref: /external/autotest/client/site_tests/longevity_Tracker/longevity_Tracker.py

# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# pylint: disable=module-missing-docstring
# pylint: disable=docstring-section-name

import csv
import glob
import httplib
import json
import logging
import os
import re
import shutil
import time
import urllib
import urllib2

from autotest_lib.client.bin import test
from autotest_lib.client.bin import utils
from autotest_lib.client.common_lib import error
from autotest_lib.client.cros import constants

# TODO(scunningham): Return to 72000 (20 hrs) after server-side stabilizes.
TEST_DURATION = 10800  # Duration of test (3 hrs) in seconds.
SAMPLE_INTERVAL = 60  # Length of measurement samples in seconds.
METRIC_INTERVAL = 3600  # Length between metric calculation in seconds.
STABILIZATION_DURATION = 60  # Time for test stabilization in seconds.
TMP_DIRECTORY = '/tmp/'
EXIT_FLAG_FILE = TMP_DIRECTORY + 'longevity_terminate'
PERF_FILE_NAME_PREFIX = 'perf'
OLD_FILE_AGE = 14400  # Age of old files to be deleted in minutes = 10 days.
# The manifest.json file for a Chrome Extension contains the app name, id,
# version, and other app info. It is accessible by the OS only when the app
# is running, and thus it's cryptohome directory mounted. Only one Kiosk app
# can be running at a time.
MANIFEST_PATTERN = '/home/.shadow/*/mount/user/Extensions/%s/*/manifest.json'
VERSION_PATTERN = r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$'
DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point'


class PerfUploadingError(Exception):
    """Exception raised in perf_uploader."""
    pass


class longevity_Tracker(test.test):
    """Monitor device and App stability over long periods of time."""

    version = 1

    def initialize(self):
        self.temp_dir = os.path.split(self.tmpdir)[0]

    def _get_cpu_usage(self):
        """Compute percent CPU in active use over the sample interval.

        Note: This method introduces a sleep period into the test, equal to
        90% of the sample interval.

        @returns float of percent active use of CPU.
        """
        # Time between measurements is ~90% of the sample interval.
        measurement_time_delta = SAMPLE_INTERVAL * 0.90
        cpu_usage_start = utils.get_cpu_usage()
        time.sleep(measurement_time_delta)
        cpu_usage_end = utils.get_cpu_usage()
        return utils.compute_active_cpu_time(cpu_usage_start,
                                                  cpu_usage_end) * 100

    def _get_mem_usage(self):
        """Compute percent memory in active use.

        @returns float of percent memory in use.
        """
        total_memory = utils.get_mem_total()
        free_memory = utils.get_mem_free()
        return ((total_memory - free_memory) / total_memory) * 100

    def _get_max_temperature(self):
        """Get temperature of hottest sensor in Celsius.

        @returns float of temperature of hottest sensor.
        """
        temperature = utils.get_current_temperature_max()
        if not temperature:
            temperature = 0
        return temperature

    def _get_hwid(self):
        """Get hwid of test device, e.g., 'WOLF C4A-B2B-A47'.

        @returns string of hwid (Hardware ID) of device under test.
        """
        with os.popen('crossystem hwid 2>/dev/null', 'r') as hwid_proc:
            hwid = hwid_proc.read()
        if not hwid:
            hwid = 'undefined'
        return hwid

    def elapsed_time(self, mark_time):
        """Get time elapsed since |mark_time|.

        @param mark_time: point in time from which elapsed time is measured.
        @returns time elapsed since the marked time.
        """
        return time.time() - mark_time

    def modulo_time(self, timer, interval):
        """Get time eplased on |timer| for the |interval| modulus.

        Value returned is used to adjust the timer so that it is synchronized
        with the current interval.

        @param timer: time on timer, in seconds.
        @param interval: period of time in seconds.
        @returns time elapsed from the start of the current interval.
        """
        return timer % int(interval)

    def syncup_time(self, timer, interval):
        """Get time remaining on |timer| for the |interval| modulus.

        Value returned is used to induce sleep just long enough to put the
        process back in sync with the timer.

        @param timer: time on timer, in seconds.
        @param interval: period of time in seconds.
        @returns time remaining till the end of the current interval.
        """
        return interval - (timer % int(interval))

    def _record_perf_measurements(self, perf_values, perf_writer):
        """Record attribute performance measurements, and write to file.

        @param perf_values: dict of attribute performance values.
        @param perf_writer: file to write performance measurements.
        """
        # Get performance measurements.
        cpu_usage = '%.3f' % self._get_cpu_usage()
        mem_usage = '%.3f' % self._get_mem_usage()
        max_temp = '%.3f' % self._get_max_temperature()

        # Append measurements to attribute lists in perf values dictionary.
        perf_values['cpu'].append(cpu_usage)
        perf_values['mem'].append(mem_usage)
        perf_values['temp'].append(max_temp)

        # Write performance measurements to perf timestamped file.
        time_stamp = time.strftime('%Y/%m/%d %H:%M:%S')
        perf_writer.writerow([time_stamp, cpu_usage, mem_usage, max_temp])
        logging.info('Time: %s, CPU: %s, Mem: %s, Temp: %s',
                     time_stamp, cpu_usage, mem_usage, max_temp)

    def _record_90th_metrics(self, perf_values, perf_metrics):
        """Record 90th percentile metric of attribute performance values.

        @param perf_values: dict attribute performance values.
        @param perf_metrics: dict attribute 90%-ile performance metrics.
        """
        # Calculate 90th percentile for each attribute.
        cpu_values = perf_values['cpu']
        mem_values = perf_values['mem']
        temp_values = perf_values['temp']
        cpu_metric = sorted(cpu_values)[(len(cpu_values) * 9) // 10]
        mem_metric = sorted(mem_values)[(len(mem_values) * 9) // 10]
        temp_metric = sorted(temp_values)[(len(temp_values) * 9) // 10]
        logging.info('== Performance values: %s', perf_values)
        logging.info('== 90th percentile: cpu: %s, mem: %s, temp: %s',
                     cpu_metric, mem_metric, temp_metric)

        # Append 90th percentile to each attribute performance metric.
        perf_metrics['cpu'].append(cpu_metric)
        perf_metrics['mem'].append(mem_metric)
        perf_metrics['temp'].append(temp_metric)

    def _get_median_metrics(self, metrics):
        """Returns median of each attribute performance metric.

        If no metric values were recorded, return 0 for each metric.

        @param metrics: dict of attribute performance metric lists.
        @returns dict of attribute performance metric medians.
        """
        if len(metrics['cpu']):
            cpu_metric = sorted(metrics['cpu'])[len(metrics['cpu']) // 2]
            mem_metric = sorted(metrics['mem'])[len(metrics['mem']) // 2]
            temp_metric = sorted(metrics['temp'])[len(metrics['temp']) // 2]
        else:
            cpu_metric = 0
            mem_metric = 0
            temp_metric = 0
        logging.info('== Median: cpu: %s, mem: %s, temp: %s',
                     cpu_metric, mem_metric, temp_metric)
        return {'cpu': cpu_metric, 'mem': mem_metric, 'temp': temp_metric}

    def _append_to_aggregated_file(self, ts_file, ag_file):
        """Append contents of perf timestamp file to perf aggregated file.

        @param ts_file: file handle for performance timestamped file.
        @param ag_file: file handle for performance aggregated file.
        """
        next(ts_file)  # Skip fist line (the header) of timestamped file.
        for line in ts_file:
            ag_file.write(line)

    def _copy_aggregated_to_resultsdir(self, aggregated_fpath):
        """Copy perf aggregated file to results dir for AutoTest results.

        Note: The AutoTest results default directory is located at /usr/local/
        autotest/results/default/longevity_Tracker/results

        @param aggregated_fpath: file path to Aggregated performance values.
        """
        results_fpath = os.path.join(self.resultsdir, 'perf.csv')
        shutil.copy(aggregated_fpath, results_fpath)
        logging.info('Copied %s to %s)', aggregated_fpath, results_fpath)

    def _write_perf_keyvals(self, perf_results):
        """Write perf results to keyval file for AutoTest results.

        @param perf_results: dict of attribute performance metrics.
        """
        perf_keyval = {}
        perf_keyval['cpu_usage'] = perf_results['cpu']
        perf_keyval['memory_usage'] = perf_results['mem']
        perf_keyval['temperature'] = perf_results['temp']
        self.write_perf_keyval(perf_keyval)

    def _write_perf_results(self, perf_results):
        """Write perf results to results-chart.json file for Perf Dashboard.

        @param perf_results: dict of attribute performance metrics.
        """
        cpu_metric = perf_results['cpu']
        mem_metric = perf_results['mem']
        ec_metric = perf_results['temp']
        self.output_perf_value(description='cpu_usage', value=cpu_metric,
                               units='%', higher_is_better=False)
        self.output_perf_value(description='mem_usage', value=mem_metric,
                               units='%', higher_is_better=False)
        self.output_perf_value(description='max_temp', value=ec_metric,
                               units='Celsius', higher_is_better=False)

    def _read_perf_results(self):
        """Read perf results from results-chart.json file for Perf Dashboard.

        @returns dict of perf results, formatted as JSON chart data.
        """
        results_file = os.path.join(self.resultsdir, 'results-chart.json')
        with open(results_file, 'r') as fp:
            contents = fp.read()
            chart_data = json.loads(contents)
        return chart_data

    def _get_point_id(self, cros_version, epoch_minutes):
        """Compute point ID from ChromeOS version number and epoch minutes.

        @param cros_version: String of ChromeOS version number.
        @param epoch_minutes: String of minutes since 1970.

        @return unique integer ID computed from given version and epoch.
        """
        # Number of digits from each part of the Chrome OS version string.
        cros_version_col_widths = [0, 4, 3, 2]

        def get_digits(version_num, column_widths):
            if re.match(VERSION_PATTERN, version_num):
                computed_string = ''
                version_parts = version_num.split('.')
                for i, version_part in enumerate(version_parts):
                    if column_widths[i]:
                        computed_string += version_part.zfill(column_widths[i])
                return computed_string
            else:
                return None

        cros_digits = get_digits(cros_version, cros_version_col_widths)
        epoch_digits = epoch_minutes[-8:]
        if not cros_digits:
            return None
        return int(epoch_digits + cros_digits)

    def _get_kiosk_app_info(self, app_id):
        """Get kiosk app name and version from manifest.json file.

        Get the Kiosk App name and version strings from the manifest file of
        the specified |app_id| Extension in the currently running session. If
        |app_id| is empty or None, then return 'none' for the kiosk app info.

        Raise an error if no manifest is found (ie, |app_id| is not running),
        or if multiple manifest files are found (ie, |app_id| is running, but
        the |app_id| dir contains multiple versions or manifest files).

        @param app_id: string kiosk application identification.
        @returns dict of Kiosk name and version number strings.
        @raises: An error.TestError if single manifest is not found.
        """
        kiosk_app_info = {'name': 'none', 'version': 'none'}
        if not app_id:
            return kiosk_app_info

        # Get path to manifest file of the running Kiosk app_id.
        app_manifest_pattern = (MANIFEST_PATTERN % app_id)
        logging.info('app_manifest_pattern: %s', app_manifest_pattern)
        file_paths = glob.glob(app_manifest_pattern)
        # Raise error if current session has no Kiosk Apps running.
        if len(file_paths) == 0:
            raise error.TestError('Kiosk App ID=%s is not running.' % app_id)
        # Raise error if running Kiosk App has multiple manifest files.
        if len(file_paths) > 1:
            raise error.TestError('Kiosk App ID=%s has multiple manifest '
                                  'files.' % app_id)
        kiosk_manifest = open(file_paths[0], 'r').read()
        manifest_json = json.loads(kiosk_manifest)
        # If manifest is missing name or version key, set to 'undefined'.
        kiosk_app_info['name'] = manifest_json.get('name', 'undefined')
        kiosk_app_info['version'] = manifest_json.get('version', 'undefined')
        return kiosk_app_info

    def _format_data_for_upload(self, chart_data):
        """Collect chart data into an uploadable data JSON object.

        @param chart_data: performance results formatted as chart data.
        """
        perf_values = {
            'format_version': '1.0',
            'benchmark_name': self.test_suite_name,
            'charts': chart_data,
        }

        dash_entry = {
            'master': 'ChromeOS_Enterprise',
            'bot': 'cros-%s' % self.board_name,
            'point_id': self.point_id,
            'versions': {
                'cros_version': self.chromeos_version,
                'chrome_version': self.chrome_version,
            },
            'supplemental': {
                'default_rev': 'r_cros_version',
                'hardware_identifier': 'a_' + self.hw_id,
                'kiosk_app_name': 'a_' + self.kiosk_app_name,
                'kiosk_app_version': 'r_' + self.kiosk_app_version
            },
            'chart_data': perf_values
        }
        return {'data': json.dumps(dash_entry)}

    def _send_to_dashboard(self, data_obj):
        """Send formatted perf data to the perf dashboard.

        @param data_obj: data object as returned by _format_data_for_upload().

        @raises PerfUploadingError if an exception was raised when uploading.
        """
        logging.debug('data_obj: %s', data_obj)
        encoded = urllib.urlencode(data_obj)
        req = urllib2.Request(DASHBOARD_UPLOAD_URL, encoded)
        try:
            urllib2.urlopen(req)
        except urllib2.HTTPError as e:
            raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' %
                                     (e.code, e.msg, data_obj['data']))
        except urllib2.URLError as e:
            raise PerfUploadingError('URLError: %s for JSON %s\n' %
                                     (str(e.reason), data_obj['data']))
        except httplib.HTTPException:
            raise PerfUploadingError('HTTPException for JSON %s\n' %
                                     data_obj['data'])

    def _get_chrome_version(self):
        """Get the Chrome version number and milestone as strings.

        Invoke "chrome --version" to get the version number and milestone.

        @return A tuple (chrome_ver, milestone) where "chrome_ver" is the
            current Chrome version number as a string (in the form "W.X.Y.Z")
            and "milestone" is the first component of the version number
            (the "W" from "W.X.Y.Z").  If the version number cannot be parsed
            in the "W.X.Y.Z" format, the "chrome_ver" will be the full output
            of "chrome --version" and the milestone will be the empty string.
        """
        chrome_version = utils.system_output(constants.CHROME_VERSION_COMMAND,
                                             ignore_status=True)
        chrome_version = utils.parse_chrome_version(chrome_version)
        return chrome_version

    def _open_perf_file(self, file_path):
        """Open a perf file. Write header line if new. Return file object.

        If the file on |file_path| already exists, then open file for
        appending only. Otherwise open for writing only.

        @param file_path: file path for perf file.
        @returns file object for the perf file.
        """
        # If file exists, open it for appending. Do not write header.
        if os.path.isfile(file_path):
            perf_file = open(file_path, 'a+')
        # Otherwise, create it for writing. Write header on first line.
        else:
            perf_file = open(file_path, 'w')  # Erase if existing file.
            perf_file.write('Time,CPU,Memory,Temperature (C)\r\n')
        return perf_file

    def _run_test_cycle(self):
        """Track performance of Chrome OS over a long period of time.

        This method collects performance measurements, and calculates metrics
        to upload to the performance dashboard. It creates two files to
        collect and store performance values and results: perf_<timestamp>.csv
        and perf_aggregated.csv.

        At the start, it creates a unique perf timestamped file in the test's
        temp_dir. As the cycle runs, it saves a time-stamped performance
        value after each sample interval. Periodically, it calculates
        the 90th percentile performance metrics from these values.

        The perf_<timestamp> files on the device will survive multiple runs
        of the longevity_Tracker by the server-side test, and will also
        survive multiple runs of the server-side test. The script will
        delete them after 10 days, to prevent filling up the SSD.

        At the end, it opens the perf aggregated file in the test's temp_dir,
        and appends the contents of the perf timestamped file. It then
        copies the perf aggregated file to the results directory as perf.csv.
        This perf.csv file will be consumed by the AutoTest backend when the
        server-side test ends.

        Note that the perf_aggregated.csv file will grow larger with each run
        of longevity_Tracker on the device by the server-side test. However,
        the server-side test will delete file in the end.

        This method also calculates 90th percentile and median metrics, and
        returns the median metrics. Median metrics will be pushed to the perf
        dashboard with a unique point_id.

        @returns list of median performance metrics.
        """
        # Allow system to stabilize before start taking measurements.
        test_start_time = time.time()
        time.sleep(STABILIZATION_DURATION)

        perf_values = {'cpu': [], 'mem': [], 'temp': []}
        perf_metrics = {'cpu': [], 'mem': [], 'temp': []}

        # Create perf_<timestamp> file and writer.
        timestamp_fname = (PERF_FILE_NAME_PREFIX +
                           time.strftime('_%Y-%m-%d_%H-%M') + '.csv')
        timestamp_fpath = os.path.join(self.temp_dir, timestamp_fname)
        timestamp_file = self._open_perf_file(timestamp_fpath)
        timestamp_writer = csv.writer(timestamp_file)

        # Align time of loop start with the sample interval.
        test_elapsed_time = self.elapsed_time(test_start_time)
        time.sleep(self.syncup_time(test_elapsed_time, SAMPLE_INTERVAL))
        test_elapsed_time = self.elapsed_time(test_start_time)

        metric_start_time = time.time()
        metric_prev_time = metric_start_time

        metric_elapsed_prev_time = self.elapsed_time(metric_prev_time)
        offset = self.modulo_time(metric_elapsed_prev_time, METRIC_INTERVAL)
        metric_timer = metric_elapsed_prev_time + offset
        while self.elapsed_time(test_start_time) <= TEST_DURATION:
            if os.path.isfile(EXIT_FLAG_FILE):
                logging.info('Exit flag file detected. Exiting test.')
                break
            self._record_perf_measurements(perf_values, timestamp_writer)

            # Periodically calculate and record 90th percentile metrics.
            metric_elapsed_prev_time = self.elapsed_time(metric_prev_time)
            metric_timer = metric_elapsed_prev_time + offset
            if metric_timer >= METRIC_INTERVAL:
                self._record_90th_metrics(perf_values, perf_metrics)
                perf_values = {'cpu': [], 'mem': [], 'temp': []}

                # Set previous time to current time.
                metric_prev_time = time.time()
                metric_elapsed_prev_time = self.elapsed_time(metric_prev_time)

                # Calculate offset based on the original start time.
                metric_elapsed_time = self.elapsed_time(metric_start_time)
                offset = self.modulo_time(metric_elapsed_time, METRIC_INTERVAL)

                # Set the timer to time elapsed plus offset to next interval.
                metric_timer = metric_elapsed_prev_time + offset

            # Sync the loop time to the sample interval.
            test_elapsed_time = self.elapsed_time(test_start_time)
            time.sleep(self.syncup_time(test_elapsed_time, SAMPLE_INTERVAL))

        # Close perf timestamp file.
        timestamp_file.close()

        # Open perf timestamp file to read, and aggregated file to append.
        timestamp_file = open(timestamp_fpath, 'r')
        aggregated_fname = (PERF_FILE_NAME_PREFIX + '_aggregated.csv')
        aggregated_fpath = os.path.join(self.temp_dir, aggregated_fname)
        aggregated_file = self._open_perf_file(aggregated_fpath)

        # Append contents of perf timestamp file to perf aggregated file.
        self._append_to_aggregated_file(timestamp_file, aggregated_file)
        timestamp_file.close()
        aggregated_file.close()

        # Copy perf aggregated file to test results directory.
        self._copy_aggregated_to_resultsdir(aggregated_fpath)

        # Return median of each attribute performance metric.
        return self._get_median_metrics(perf_metrics)

    def run_once(self, kiosk_app_attributes=None):
        if kiosk_app_attributes:
            app_name, app_id, ext_page = (
                kiosk_app_attributes.rstrip().split(':'))
        self.subtest_name = app_name
        self.board_name = utils.get_board()
        self.hw_id = self._get_hwid()
        self.chrome_version = self._get_chrome_version()[0]
        self.chromeos_version = '0.' + utils.get_chromeos_release_version()
        self.epoch_minutes = str(int(time.time() / 60))  # Minutes since 1970.
        self.point_id = self._get_point_id(self.chromeos_version,
                                           self.epoch_minutes)

        kiosk_info = self._get_kiosk_app_info(app_id)
        self.kiosk_app_name = kiosk_info['name']
        self.kiosk_app_version = kiosk_info['version']
        self.test_suite_name = self.tagged_testname
        if self.subtest_name:
            self.test_suite_name += '.' + self.subtest_name

        # Delete exit flag file at start of test run.
        if os.path.isfile(EXIT_FLAG_FILE):
            os.remove(EXIT_FLAG_FILE)

        # Run a single test cycle.
        self.perf_results = {'cpu': '0', 'mem': '0', 'temp': '0'}
        self.perf_results = self._run_test_cycle()

        # Write results for AutoTest to pick up at end of test.
        self._write_perf_keyvals(self.perf_results)
        self._write_perf_results(self.perf_results)

        # Post perf results directly to performance dashboard. You may view
        # uploaded data at https://chromeperf.appspot.com/new_points,
        # with test path pattern=ChromeOS_Enterprise/cros-*/longevity*/*
        chart_data = self._read_perf_results()
        data_obj = self._format_data_for_upload(chart_data)
        self._send_to_dashboard(data_obj)

    def cleanup(self):
        """Delete aged perf data files and the exit flag file."""
        cmd = ('find %s -name %s* -type f -mmin +%s -delete' %
               (self.temp_dir, PERF_FILE_NAME_PREFIX, OLD_FILE_AGE))
        os.system(cmd)
        if os.path.isfile(EXIT_FLAG_FILE):
            os.remove(EXIT_FLAG_FILE)