Home | History | Annotate | Download | only in common_lib
      1 #
      2 # Copyright 2008 Google Inc. Released under the GPL v2
      3 
      4 # pylint: disable-msg=C0111
      5 
      6 import StringIO
      7 import errno
      8 import itertools
      9 import logging
     10 import os
     11 import pickle
     12 import random
     13 import re
     14 import resource
     15 import select
     16 import shutil
     17 import signal
     18 import smtplib
     19 import socket
     20 import string
     21 import struct
     22 import subprocess
     23 import textwrap
     24 import time
     25 import urllib2
     26 import urlparse
     27 import warnings
     28 
     29 from threading import Thread, Event
     30 
     31 try:
     32     import hashlib
     33 except ImportError:
     34     import md5
     35     import sha
     36 
     37 from autotest_lib.client.common_lib import error, logging_manager
     38 
     39 
     40 def deprecated(func):
     41     """This is a decorator which can be used to mark functions as deprecated.
     42     It will result in a warning being emmitted when the function is used."""
     43     def new_func(*args, **dargs):
     44         warnings.warn("Call to deprecated function %s." % func.__name__,
     45                       category=DeprecationWarning)
     46         return func(*args, **dargs)
     47     new_func.__name__ = func.__name__
     48     new_func.__doc__ = func.__doc__
     49     new_func.__dict__.update(func.__dict__)
     50     return new_func
     51 
     52 
     53 class _NullStream(object):
     54     def write(self, data):
     55         pass
     56 
     57 
     58     def flush(self):
     59         pass
     60 
     61 
     62 TEE_TO_LOGS = object()
     63 _the_null_stream = _NullStream()
     64 
     65 DEFAULT_STDOUT_LEVEL = logging.DEBUG
     66 DEFAULT_STDERR_LEVEL = logging.ERROR
     67 
     68 # prefixes for logging stdout/stderr of commands
     69 STDOUT_PREFIX = '[stdout] '
     70 STDERR_PREFIX = '[stderr] '
     71 
     72 # safe characters for the shell (do not need quoting)
     73 SHELL_QUOTING_WHITELIST = frozenset(string.ascii_letters +
     74                                     string.digits +
     75                                     '_-+=')
     76 
     77 
     78 def custom_warning_handler(message, category, filename, lineno, file=None,
     79                            line=None):
     80     """Custom handler to log at the WARNING error level. Ignores |file|."""
     81     logging.warning(warnings.formatwarning(message, category, filename, lineno,
     82                                            line))
     83 
     84 warnings.showwarning = custom_warning_handler
     85 
     86 def get_stream_tee_file(stream, level, prefix=''):
     87     if stream is None:
     88         return _the_null_stream
     89     if stream is TEE_TO_LOGS:
     90         return logging_manager.LoggingFile(level=level, prefix=prefix)
     91     return stream
     92 
     93 
     94 def _join_with_nickname(base_string, nickname):
     95     if nickname:
     96         return '%s BgJob "%s" ' % (base_string, nickname)
     97     return base_string
     98 
     99 
    100 # TODO: Cleanup and possibly eliminate no_pipes, which is only used
    101 # in our master-ssh connection process, while fixing underlying
    102 # semantics problem in BgJob. See crbug.com/279312
    103 class BgJob(object):
    104     def __init__(self, command, stdout_tee=None, stderr_tee=None, verbose=True,
    105                  stdin=None, stderr_level=DEFAULT_STDERR_LEVEL, nickname=None,
    106                  no_pipes=False):
    107         """Create and start a new BgJob.
    108 
    109         This constructor creates a new BgJob, and uses Popen to start a new
    110         subprocess with given command. It returns without blocking on execution
    111         of the subprocess.
    112 
    113         After starting a new BgJob, use output_prepare to connect the process's
    114         stdout and stderr pipes to the stream of your choice.
    115 
    116         When the job is running, the jobs's output streams are only read from
    117         when process_output is called.
    118 
    119         @param command: command to be executed in new subprocess. May be either
    120                         a list, or a string (in which case Popen will be called
    121                         with shell=True)
    122         @param stdout_tee: Optional additional stream that the process's stdout
    123                            stream output will be written to. Or, specify
    124                            base_utils.TEE_TO_LOGS and the output will handled by
    125                            the standard logging_manager.
    126         @param stderr_tee: Same as stdout_tee, but for stderr.
    127         @param verbose: Boolean, make BgJob logging more verbose.
    128         @param stdin: Stream object, will be passed to Popen as the new
    129                       process's stdin.
    130         @param stderr_level: A logging level value. If stderr_tee was set to
    131                              base_utils.TEE_TO_LOGS, sets the level that tee'd
    132                              stderr output will be logged at. Ignored
    133                              otherwise.
    134         @param nickname: Optional string, to be included in logging messages
    135         @param no_pipes: Boolean, default False. If True, this subprocess
    136                          created by this BgJob does NOT use subprocess.PIPE
    137                          for its stdin or stderr streams. Instead, these
    138                          streams are connected to the logging manager
    139                          (regardless of the values of stdout_tee and
    140                          stderr_tee).
    141                          If no_pipes is True, then calls to output_prepare,
    142                          process_output, and cleanup will result in an
    143                          InvalidBgJobCall exception. no_pipes should be
    144                          True for BgJobs that do not interact via stdout/stderr
    145                          with other BgJobs, or long runing background jobs that
    146                          will never be joined with join_bg_jobs, such as the
    147                          master-ssh connection BgJob.
    148         """
    149         self.command = command
    150         self._no_pipes = no_pipes
    151         if no_pipes:
    152             stdout_tee = TEE_TO_LOGS
    153             stderr_tee = TEE_TO_LOGS
    154         self.stdout_tee = get_stream_tee_file(stdout_tee, DEFAULT_STDOUT_LEVEL,
    155                 prefix=_join_with_nickname(STDOUT_PREFIX, nickname))
    156         self.stderr_tee = get_stream_tee_file(stderr_tee, stderr_level,
    157                 prefix=_join_with_nickname(STDERR_PREFIX, nickname))
    158         self.result = CmdResult(command)
    159 
    160         # allow for easy stdin input by string, we'll let subprocess create
    161         # a pipe for stdin input and we'll write to it in the wait loop
    162         if isinstance(stdin, basestring):
    163             self.string_stdin = stdin
    164             stdin = subprocess.PIPE
    165         else:
    166             self.string_stdin = None
    167 
    168 
    169         if no_pipes:
    170             stdout_param = self.stdout_tee
    171             stderr_param = self.stderr_tee
    172         else:
    173             stdout_param = subprocess.PIPE
    174             stderr_param = subprocess.PIPE
    175 
    176         if verbose:
    177             logging.debug("Running '%s'", command)
    178         if type(command) == list:
    179             self.sp = subprocess.Popen(command,
    180                                        stdout=stdout_param,
    181                                        stderr=stderr_param,
    182                                        preexec_fn=self._reset_sigpipe,
    183                                        stdin=stdin)
    184         else:
    185             self.sp = subprocess.Popen(command, stdout=stdout_param,
    186                                        stderr=stderr_param,
    187                                        preexec_fn=self._reset_sigpipe, shell=True,
    188                                        executable="/bin/bash",
    189                                        stdin=stdin)
    190 
    191         self._output_prepare_called = False
    192         self._process_output_warned = False
    193         self._cleanup_called = False
    194         self.stdout_file = _the_null_stream
    195         self.stderr_file = _the_null_stream
    196 
    197     def output_prepare(self, stdout_file=_the_null_stream,
    198                        stderr_file=_the_null_stream):
    199         """Connect the subprocess's stdout and stderr to streams.
    200 
    201         Subsequent calls to output_prepare are permitted, and will reassign
    202         the streams. However, this will have the side effect that the ultimate
    203         call to cleanup() will only remember the stdout and stderr data up to
    204         the last output_prepare call when saving this data to BgJob.result.
    205 
    206         @param stdout_file: Stream that output from the process's stdout pipe
    207                             will be written to. Default: a null stream.
    208         @param stderr_file: Stream that output from the process's stdout pipe
    209                             will be written to. Default: a null stream.
    210         """
    211         if self._no_pipes:
    212             raise error.InvalidBgJobCall('Cannot call output_prepare on a '
    213                                          'job with no_pipes=True.')
    214         if self._output_prepare_called:
    215             logging.warning('BgJob [%s] received a duplicate call to '
    216                             'output prepare. Allowing, but this may result '
    217                             'in data missing from BgJob.result.')
    218         self.stdout_file = stdout_file
    219         self.stderr_file = stderr_file
    220         self._output_prepare_called = True
    221 
    222 
    223     def process_output(self, stdout=True, final_read=False):
    224         """Read from process's output stream, and write data to destinations.
    225 
    226         This function reads up to 1024 bytes from the background job's
    227         stdout or stderr stream, and writes the resulting data to the BgJob's
    228         output tee and to the stream set up in output_prepare.
    229 
    230         Warning: Calls to process_output will block on reads from the
    231         subprocess stream, and will block on writes to the configured
    232         destination stream.
    233 
    234         @param stdout: True = read and process data from job's stdout.
    235                        False = from stderr.
    236                        Default: True
    237         @param final_read: Do not read only 1024 bytes from stream. Instead,
    238                            read and process all data until end of the stream.
    239 
    240         """
    241         if self._no_pipes:
    242             raise error.InvalidBgJobCall('Cannot call process_output on '
    243                                          'a job with no_pipes=True')
    244         if not self._output_prepare_called and not self._process_output_warned:
    245             logging.warning('BgJob with command [%s] handled a process_output '
    246                             'call before output_prepare was called. '
    247                             'Some output data discarded. '
    248                             'Future warnings suppressed.',
    249                             self.command)
    250             self._process_output_warned = True
    251         if stdout:
    252             pipe, buf, tee = self.sp.stdout, self.stdout_file, self.stdout_tee
    253         else:
    254             pipe, buf, tee = self.sp.stderr, self.stderr_file, self.stderr_tee
    255 
    256         if final_read:
    257             # read in all the data we can from pipe and then stop
    258             data = []
    259             while select.select([pipe], [], [], 0)[0]:
    260                 data.append(os.read(pipe.fileno(), 1024))
    261                 if len(data[-1]) == 0:
    262                     break
    263             data = "".join(data)
    264         else:
    265             # perform a single read
    266             data = os.read(pipe.fileno(), 1024)
    267         buf.write(data)
    268         tee.write(data)
    269 
    270 
    271     def cleanup(self):
    272         """Clean up after BgJob.
    273 
    274         Flush the stdout_tee and stderr_tee buffers, close the
    275         subprocess stdout and stderr buffers, and saves data from
    276         the configured stdout and stderr destination streams to
    277         self.result. Duplicate calls ignored with a warning.
    278         """
    279         if self._no_pipes:
    280             raise error.InvalidBgJobCall('Cannot call cleanup on '
    281                                          'a job with no_pipes=True')
    282         if self._cleanup_called:
    283             logging.warning('BgJob [%s] received a duplicate call to '
    284                             'cleanup. Ignoring.', self.command)
    285             return
    286         try:
    287             self.stdout_tee.flush()
    288             self.stderr_tee.flush()
    289             self.sp.stdout.close()
    290             self.sp.stderr.close()
    291             self.result.stdout = self.stdout_file.getvalue()
    292             self.result.stderr = self.stderr_file.getvalue()
    293         finally:
    294             self._cleanup_called = True
    295 
    296 
    297     def _reset_sigpipe(self):
    298         signal.signal(signal.SIGPIPE, signal.SIG_DFL)
    299 
    300 
    301 def ip_to_long(ip):
    302     # !L is a long in network byte order
    303     return struct.unpack('!L', socket.inet_aton(ip))[0]
    304 
    305 
    306 def long_to_ip(number):
    307     # See above comment.
    308     return socket.inet_ntoa(struct.pack('!L', number))
    309 
    310 
    311 def create_subnet_mask(bits):
    312     return (1 << 32) - (1 << 32-bits)
    313 
    314 
    315 def format_ip_with_mask(ip, mask_bits):
    316     masked_ip = ip_to_long(ip) & create_subnet_mask(mask_bits)
    317     return "%s/%s" % (long_to_ip(masked_ip), mask_bits)
    318 
    319 
    320 def normalize_hostname(alias):
    321     ip = socket.gethostbyname(alias)
    322     return socket.gethostbyaddr(ip)[0]
    323 
    324 
    325 def get_ip_local_port_range():
    326     match = re.match(r'\s*(\d+)\s*(\d+)\s*$',
    327                      read_one_line('/proc/sys/net/ipv4/ip_local_port_range'))
    328     return (int(match.group(1)), int(match.group(2)))
    329 
    330 
    331 def set_ip_local_port_range(lower, upper):
    332     write_one_line('/proc/sys/net/ipv4/ip_local_port_range',
    333                    '%d %d\n' % (lower, upper))
    334 
    335 
    336 def send_email(mail_from, mail_to, subject, body):
    337     """
    338     Sends an email via smtp
    339 
    340     mail_from: string with email address of sender
    341     mail_to: string or list with email address(es) of recipients
    342     subject: string with subject of email
    343     body: (multi-line) string with body of email
    344     """
    345     if isinstance(mail_to, str):
    346         mail_to = [mail_to]
    347     msg = "From: %s\nTo: %s\nSubject: %s\n\n%s" % (mail_from, ','.join(mail_to),
    348                                                    subject, body)
    349     try:
    350         mailer = smtplib.SMTP('localhost')
    351         try:
    352             mailer.sendmail(mail_from, mail_to, msg)
    353         finally:
    354             mailer.quit()
    355     except Exception, e:
    356         # Emails are non-critical, not errors, but don't raise them
    357         print "Sending email failed. Reason: %s" % repr(e)
    358 
    359 
    360 def read_one_line(filename):
    361     return open(filename, 'r').readline().rstrip('\n')
    362 
    363 
    364 def read_file(filename):
    365     f = open(filename)
    366     try:
    367         return f.read()
    368     finally:
    369         f.close()
    370 
    371 
    372 def get_field(data, param, linestart="", sep=" "):
    373     """
    374     Parse data from string.
    375     @param data: Data to parse.
    376         example:
    377           data:
    378              cpu   324 345 34  5 345
    379              cpu0  34  11  34 34  33
    380              ^^^^
    381              start of line
    382              params 0   1   2  3   4
    383     @param param: Position of parameter after linestart marker.
    384     @param linestart: String to which start line with parameters.
    385     @param sep: Separator between parameters regular expression.
    386     """
    387     search = re.compile(r"(?<=^%s)\s*(.*)" % linestart, re.MULTILINE)
    388     find = search.search(data)
    389     if find != None:
    390         return re.split("%s" % sep, find.group(1))[param]
    391     else:
    392         print "There is no line which starts with %s in data." % linestart
    393         return None
    394 
    395 
    396 def write_one_line(filename, line):
    397     open_write_close(filename, str(line).rstrip('\n') + '\n')
    398 
    399 
    400 def open_write_close(filename, data):
    401     f = open(filename, 'w')
    402     try:
    403         f.write(data)
    404     finally:
    405         f.close()
    406 
    407 
    408 def locate_file(path, base_dir=None):
    409     """Locates a file.
    410 
    411     @param path: The path of the file being located. Could be absolute or relative
    412         path. For relative path, it tries to locate the file from base_dir.
    413     @param base_dir (optional): Base directory of the relative path.
    414 
    415     @returns Absolute path of the file if found. None if path is None.
    416     @raises error.TestFail if the file is not found.
    417     """
    418     if path is None:
    419         return None
    420 
    421     if not os.path.isabs(path) and base_dir is not None:
    422         # Assume the relative path is based in autotest directory.
    423         path = os.path.join(base_dir, path)
    424     if not os.path.isfile(path):
    425         raise error.TestFail('ERROR: Unable to find %s' % path)
    426     return path
    427 
    428 
    429 def matrix_to_string(matrix, header=None):
    430     """
    431     Return a pretty, aligned string representation of a nxm matrix.
    432 
    433     This representation can be used to print any tabular data, such as
    434     database results. It works by scanning the lengths of each element
    435     in each column, and determining the format string dynamically.
    436 
    437     @param matrix: Matrix representation (list with n rows of m elements).
    438     @param header: Optional tuple or list with header elements to be displayed.
    439     """
    440     if type(header) is list:
    441         header = tuple(header)
    442     lengths = []
    443     if header:
    444         for column in header:
    445             lengths.append(len(column))
    446     for row in matrix:
    447         for i, column in enumerate(row):
    448             column = unicode(column).encode("utf-8")
    449             cl = len(column)
    450             try:
    451                 ml = lengths[i]
    452                 if cl > ml:
    453                     lengths[i] = cl
    454             except IndexError:
    455                 lengths.append(cl)
    456 
    457     lengths = tuple(lengths)
    458     format_string = ""
    459     for length in lengths:
    460         format_string += "%-" + str(length) + "s "
    461     format_string += "\n"
    462 
    463     matrix_str = ""
    464     if header:
    465         matrix_str += format_string % header
    466     for row in matrix:
    467         matrix_str += format_string % tuple(row)
    468 
    469     return matrix_str
    470 
    471 
    472 def read_keyval(path, type_tag=None):
    473     """
    474     Read a key-value pair format file into a dictionary, and return it.
    475     Takes either a filename or directory name as input. If it's a
    476     directory name, we assume you want the file to be called keyval.
    477 
    478     @param path: Full path of the file to read from.
    479     @param type_tag: If not None, only keyvals with key ending
    480                      in a suffix {type_tag} will be collected.
    481     """
    482     if os.path.isdir(path):
    483         path = os.path.join(path, 'keyval')
    484     if not os.path.exists(path):
    485         return {}
    486 
    487     if type_tag:
    488         pattern = r'^([-\.\w]+)\{%s\}=(.*)$' % type_tag
    489     else:
    490         pattern = r'^([-\.\w]+)=(.*)$'
    491 
    492     keyval = {}
    493     f = open(path)
    494     for line in f:
    495         line = re.sub('#.*', '', line).rstrip()
    496         if not line:
    497             continue
    498         match = re.match(pattern, line)
    499         if match:
    500             key = match.group(1)
    501             value = match.group(2)
    502             if re.search('^\d+$', value):
    503                 value = int(value)
    504             elif re.search('^(\d+\.)?\d+$', value):
    505                 value = float(value)
    506             keyval[key] = value
    507         else:
    508             raise ValueError('Invalid format line: %s' % line)
    509     f.close()
    510     return keyval
    511 
    512 
    513 def write_keyval(path, dictionary, type_tag=None, tap_report=None):
    514     """
    515     Write a key-value pair format file out to a file. This uses append
    516     mode to open the file, so existing text will not be overwritten or
    517     reparsed.
    518 
    519     If type_tag is None, then the key must be composed of alphanumeric
    520     characters (or dashes+underscores). However, if type-tag is not
    521     null then the keys must also have "{type_tag}" as a suffix. At
    522     the moment the only valid values of type_tag are "attr" and "perf".
    523 
    524     @param path: full path of the file to be written
    525     @param dictionary: the items to write
    526     @param type_tag: see text above
    527     """
    528     if os.path.isdir(path):
    529         path = os.path.join(path, 'keyval')
    530     keyval = open(path, 'a')
    531 
    532     if type_tag is None:
    533         key_regex = re.compile(r'^[-\.\w]+$')
    534     else:
    535         if type_tag not in ('attr', 'perf'):
    536             raise ValueError('Invalid type tag: %s' % type_tag)
    537         escaped_tag = re.escape(type_tag)
    538         key_regex = re.compile(r'^[-\.\w]+\{%s\}$' % escaped_tag)
    539     try:
    540         for key in sorted(dictionary.keys()):
    541             if not key_regex.search(key):
    542                 raise ValueError('Invalid key: %s' % key)
    543             keyval.write('%s=%s\n' % (key, dictionary[key]))
    544     finally:
    545         keyval.close()
    546 
    547     # same for tap
    548     if tap_report is not None and tap_report.do_tap_report:
    549         tap_report.record_keyval(path, dictionary, type_tag=type_tag)
    550 
    551 class FileFieldMonitor(object):
    552     """
    553     Monitors the information from the file and reports it's values.
    554 
    555     It gather the information at start and stop of the measurement or
    556     continuously during the measurement.
    557     """
    558     class Monitor(Thread):
    559         """
    560         Internal monitor class to ensure continuous monitor of monitored file.
    561         """
    562         def __init__(self, master):
    563             """
    564             @param master: Master class which control Monitor
    565             """
    566             Thread.__init__(self)
    567             self.master = master
    568 
    569         def run(self):
    570             """
    571             Start monitor in thread mode
    572             """
    573             while not self.master.end_event.isSet():
    574                 self.master._get_value(self.master.logging)
    575                 time.sleep(self.master.time_step)
    576 
    577 
    578     def __init__(self, status_file, data_to_read, mode_diff, continuously=False,
    579                  contlogging=False, separator=" +", time_step=0.1):
    580         """
    581         Initialize variables.
    582         @param status_file: File contain status.
    583         @param mode_diff: If True make a difference of value, else average.
    584         @param data_to_read: List of tuples with data position.
    585             format: [(start_of_line,position in params)]
    586             example:
    587               data:
    588                  cpu   324 345 34  5 345
    589                  cpu0  34  11  34 34  33
    590                  ^^^^
    591                  start of line
    592                  params 0   1   2  3   4
    593         @param mode_diff: True to subtract old value from new value,
    594             False make average of the values.
    595         @parma continuously: Start the monitoring thread using the time_step
    596             as the measurement period.
    597         @param contlogging: Log data in continuous run.
    598         @param separator: Regular expression of separator.
    599         @param time_step: Time period of the monitoring value.
    600         """
    601         self.end_event = Event()
    602         self.start_time = 0
    603         self.end_time = 0
    604         self.test_time = 0
    605 
    606         self.status_file = status_file
    607         self.separator = separator
    608         self.data_to_read = data_to_read
    609         self.num_of_params = len(self.data_to_read)
    610         self.mode_diff = mode_diff
    611         self.continuously = continuously
    612         self.time_step = time_step
    613 
    614         self.value = [0 for i in range(self.num_of_params)]
    615         self.old_value = [0 for i in range(self.num_of_params)]
    616         self.log = []
    617         self.logging = contlogging
    618 
    619         self.started = False
    620         self.num_of_get_value = 0
    621         self.monitor = None
    622 
    623 
    624     def _get_value(self, logging=True):
    625         """
    626         Return current values.
    627         @param logging: If true log value in memory. There can be problem
    628           with long run.
    629         """
    630         data = read_file(self.status_file)
    631         value = []
    632         for i in range(self.num_of_params):
    633             value.append(int(get_field(data,
    634                              self.data_to_read[i][1],
    635                              self.data_to_read[i][0],
    636                              self.separator)))
    637 
    638         if logging:
    639             self.log.append(value)
    640         if not self.mode_diff:
    641             value = map(lambda x, y: x + y, value, self.old_value)
    642 
    643         self.old_value = value
    644         self.num_of_get_value += 1
    645         return value
    646 
    647 
    648     def start(self):
    649         """
    650         Start value monitor.
    651         """
    652         if self.started:
    653             self.stop()
    654         self.old_value = [0 for i in range(self.num_of_params)]
    655         self.num_of_get_value = 0
    656         self.log = []
    657         self.end_event.clear()
    658         self.start_time = time.time()
    659         self._get_value()
    660         self.started = True
    661         if (self.continuously):
    662             self.monitor = FileFieldMonitor.Monitor(self)
    663             self.monitor.start()
    664 
    665 
    666     def stop(self):
    667         """
    668         Stop value monitor.
    669         """
    670         if self.started:
    671             self.started = False
    672             self.end_time = time.time()
    673             self.test_time = self.end_time - self.start_time
    674             self.value = self._get_value()
    675             if (self.continuously):
    676                 self.end_event.set()
    677                 self.monitor.join()
    678             if (self.mode_diff):
    679                 self.value = map(lambda x, y: x - y, self.log[-1], self.log[0])
    680             else:
    681                 self.value = map(lambda x: x / self.num_of_get_value,
    682                                  self.value)
    683 
    684 
    685     def get_status(self):
    686         """
    687         @return: Status of monitored process average value,
    688             time of test and array of monitored values and time step of
    689             continuous run.
    690         """
    691         if self.started:
    692             self.stop()
    693         if self.mode_diff:
    694             for i in range(len(self.log) - 1):
    695                 self.log[i] = (map(lambda x, y: x - y,
    696                                    self.log[i + 1], self.log[i]))
    697             self.log.pop()
    698         return (self.value, self.test_time, self.log, self.time_step)
    699 
    700 
    701 def is_url(path):
    702     """Return true if path looks like a URL"""
    703     # for now, just handle http and ftp
    704     url_parts = urlparse.urlparse(path)
    705     return (url_parts[0] in ('http', 'ftp'))
    706 
    707 
    708 def urlopen(url, data=None, timeout=5):
    709     """Wrapper to urllib2.urlopen with timeout addition."""
    710 
    711     # Save old timeout
    712     old_timeout = socket.getdefaulttimeout()
    713     socket.setdefaulttimeout(timeout)
    714     try:
    715         return urllib2.urlopen(url, data=data)
    716     finally:
    717         socket.setdefaulttimeout(old_timeout)
    718 
    719 
    720 def urlretrieve(url, filename, data=None, timeout=300):
    721     """Retrieve a file from given url."""
    722     logging.debug('Fetching %s -> %s', url, filename)
    723 
    724     src_file = urlopen(url, data=data, timeout=timeout)
    725     try:
    726         dest_file = open(filename, 'wb')
    727         try:
    728             shutil.copyfileobj(src_file, dest_file)
    729         finally:
    730             dest_file.close()
    731     finally:
    732         src_file.close()
    733 
    734 
    735 def hash(type, input=None):
    736     """
    737     Returns an hash object of type md5 or sha1. This function is implemented in
    738     order to encapsulate hash objects in a way that is compatible with python
    739     2.4 and python 2.6 without warnings.
    740 
    741     Note that even though python 2.6 hashlib supports hash types other than
    742     md5 and sha1, we are artificially limiting the input values in order to
    743     make the function to behave exactly the same among both python
    744     implementations.
    745 
    746     @param input: Optional input string that will be used to update the hash.
    747     """
    748     if type not in ['md5', 'sha1']:
    749         raise ValueError("Unsupported hash type: %s" % type)
    750 
    751     try:
    752         hash = hashlib.new(type)
    753     except NameError:
    754         if type == 'md5':
    755             hash = md5.new()
    756         elif type == 'sha1':
    757             hash = sha.new()
    758 
    759     if input:
    760         hash.update(input)
    761 
    762     return hash
    763 
    764 
    765 def get_file(src, dest, permissions=None):
    766     """Get a file from src, which can be local or a remote URL"""
    767     if src == dest:
    768         return
    769 
    770     if is_url(src):
    771         urlretrieve(src, dest)
    772     else:
    773         shutil.copyfile(src, dest)
    774 
    775     if permissions:
    776         os.chmod(dest, permissions)
    777     return dest
    778 
    779 
    780 def unmap_url(srcdir, src, destdir='.'):
    781     """
    782     Receives either a path to a local file or a URL.
    783     returns either the path to the local file, or the fetched URL
    784 
    785     unmap_url('/usr/src', 'foo.tar', '/tmp')
    786                             = '/usr/src/foo.tar'
    787     unmap_url('/usr/src', 'http://site/file', '/tmp')
    788                             = '/tmp/file'
    789                             (after retrieving it)
    790     """
    791     if is_url(src):
    792         url_parts = urlparse.urlparse(src)
    793         filename = os.path.basename(url_parts[2])
    794         dest = os.path.join(destdir, filename)
    795         return get_file(src, dest)
    796     else:
    797         return os.path.join(srcdir, src)
    798 
    799 
    800 def update_version(srcdir, preserve_srcdir, new_version, install,
    801                    *args, **dargs):
    802     """
    803     Make sure srcdir is version new_version
    804 
    805     If not, delete it and install() the new version.
    806 
    807     In the preserve_srcdir case, we just check it's up to date,
    808     and if not, we rerun install, without removing srcdir
    809     """
    810     versionfile = os.path.join(srcdir, '.version')
    811     install_needed = True
    812 
    813     if os.path.exists(versionfile):
    814         old_version = pickle.load(open(versionfile))
    815         if old_version == new_version:
    816             install_needed = False
    817 
    818     if install_needed:
    819         if not preserve_srcdir and os.path.exists(srcdir):
    820             shutil.rmtree(srcdir)
    821         install(*args, **dargs)
    822         if os.path.exists(srcdir):
    823             pickle.dump(new_version, open(versionfile, 'w'))
    824 
    825 
    826 def get_stderr_level(stderr_is_expected):
    827     if stderr_is_expected:
    828         return DEFAULT_STDOUT_LEVEL
    829     return DEFAULT_STDERR_LEVEL
    830 
    831 
    832 def run(command, timeout=None, ignore_status=False,
    833         stdout_tee=None, stderr_tee=None, verbose=True, stdin=None,
    834         stderr_is_expected=None, args=(), nickname=None, ignore_timeout=False):
    835     """
    836     Run a command on the host.
    837 
    838     @param command: the command line string.
    839     @param timeout: time limit in seconds before attempting to kill the
    840             running process. The run() function will take a few seconds
    841             longer than 'timeout' to complete if it has to kill the process.
    842     @param ignore_status: do not raise an exception, no matter what the exit
    843             code of the command is.
    844     @param ignore_timeout: If True, timeouts are ignored otherwise if a
    845             timeout occurs it will raise CmdTimeoutError.
    846     @param stdout_tee: optional file-like object to which stdout data
    847             will be written as it is generated (data will still be stored
    848             in result.stdout).
    849     @param stderr_tee: likewise for stderr.
    850     @param verbose: if True, log the command being run.
    851     @param stdin: stdin to pass to the executed process (can be a file
    852             descriptor, a file object of a real file or a string).
    853     @param args: sequence of strings of arguments to be given to the command
    854             inside " quotes after they have been escaped for that; each
    855             element in the sequence will be given as a separate command
    856             argument
    857     @param nickname: Short string that will appear in logging messages
    858                      associated with this command.
    859 
    860     @return a CmdResult object or None if the command timed out and
    861             ignore_timeout is True
    862 
    863     @raise CmdError: the exit code of the command execution was not 0
    864     @raise CmdTimeoutError: the command timed out and ignore_timeout is False.
    865     """
    866     if isinstance(args, basestring):
    867         raise TypeError('Got a string for the "args" keyword argument, '
    868                         'need a sequence.')
    869 
    870     # In some cases, command will actually be a list
    871     # (For example, see get_user_hash in client/cros/cryptohome.py.)
    872     # So, to cover that case, detect if it's a string or not and convert it
    873     # into one if necessary.
    874     if not isinstance(command, basestring):
    875         command = ' '.join([sh_quote_word(arg) for arg in command])
    876 
    877     command = ' '.join([command] + [sh_quote_word(arg) for arg in args])
    878     if stderr_is_expected is None:
    879         stderr_is_expected = ignore_status
    880 
    881     try:
    882         bg_job = join_bg_jobs(
    883             (BgJob(command, stdout_tee, stderr_tee, verbose, stdin=stdin,
    884                    stderr_level=get_stderr_level(stderr_is_expected),
    885                    nickname=nickname),), timeout)[0]
    886     except error.CmdTimeoutError:
    887         if not ignore_timeout:
    888             raise
    889         return None
    890 
    891     if not ignore_status and bg_job.result.exit_status:
    892         raise error.CmdError(command, bg_job.result,
    893                              "Command returned non-zero exit status")
    894 
    895     return bg_job.result
    896 
    897 
    898 def run_parallel(commands, timeout=None, ignore_status=False,
    899                  stdout_tee=None, stderr_tee=None,
    900                  nicknames=[]):
    901     """
    902     Behaves the same as run() with the following exceptions:
    903 
    904     - commands is a list of commands to run in parallel.
    905     - ignore_status toggles whether or not an exception should be raised
    906       on any error.
    907 
    908     @return: a list of CmdResult objects
    909     """
    910     bg_jobs = []
    911     for (command, nickname) in itertools.izip_longest(commands, nicknames):
    912         bg_jobs.append(BgJob(command, stdout_tee, stderr_tee,
    913                              stderr_level=get_stderr_level(ignore_status),
    914                              nickname=nickname))
    915 
    916     # Updates objects in bg_jobs list with their process information
    917     join_bg_jobs(bg_jobs, timeout)
    918 
    919     for bg_job in bg_jobs:
    920         if not ignore_status and bg_job.result.exit_status:
    921             raise error.CmdError(command, bg_job.result,
    922                                  "Command returned non-zero exit status")
    923 
    924     return [bg_job.result for bg_job in bg_jobs]
    925 
    926 
    927 @deprecated
    928 def run_bg(command):
    929     """Function deprecated. Please use BgJob class instead."""
    930     bg_job = BgJob(command)
    931     return bg_job.sp, bg_job.result
    932 
    933 
    934 def join_bg_jobs(bg_jobs, timeout=None):
    935     """Joins the bg_jobs with the current thread.
    936 
    937     Returns the same list of bg_jobs objects that was passed in.
    938     """
    939     ret, timeout_error = 0, False
    940     for bg_job in bg_jobs:
    941         bg_job.output_prepare(StringIO.StringIO(), StringIO.StringIO())
    942 
    943     try:
    944         # We are holding ends to stdin, stdout pipes
    945         # hence we need to be sure to close those fds no mater what
    946         start_time = time.time()
    947         timeout_error = _wait_for_commands(bg_jobs, start_time, timeout)
    948 
    949         for bg_job in bg_jobs:
    950             # Process stdout and stderr
    951             bg_job.process_output(stdout=True,final_read=True)
    952             bg_job.process_output(stdout=False,final_read=True)
    953     finally:
    954         # close our ends of the pipes to the sp no matter what
    955         for bg_job in bg_jobs:
    956             bg_job.cleanup()
    957 
    958     if timeout_error:
    959         # TODO: This needs to be fixed to better represent what happens when
    960         # running in parallel. However this is backwards compatable, so it will
    961         # do for the time being.
    962         raise error.CmdTimeoutError(
    963                 bg_jobs[0].command, bg_jobs[0].result,
    964                 "Command(s) did not complete within %d seconds" % timeout)
    965 
    966 
    967     return bg_jobs
    968 
    969 
    970 def _wait_for_commands(bg_jobs, start_time, timeout):
    971     """Waits for background jobs by select polling their stdout/stderr.
    972 
    973     @param bg_jobs: A list of background jobs to wait on.
    974     @param start_time: Time used to calculate the timeout lifetime of a job.
    975     @param timeout: The timeout of the list of bg_jobs.
    976 
    977     @return: True if the return was due to a timeout, False otherwise.
    978     """
    979 
    980     # To check for processes which terminate without producing any output
    981     # a 1 second timeout is used in select.
    982     SELECT_TIMEOUT = 1
    983 
    984     read_list = []
    985     write_list = []
    986     reverse_dict = {}
    987 
    988     for bg_job in bg_jobs:
    989         read_list.append(bg_job.sp.stdout)
    990         read_list.append(bg_job.sp.stderr)
    991         reverse_dict[bg_job.sp.stdout] = (bg_job, True)
    992         reverse_dict[bg_job.sp.stderr] = (bg_job, False)
    993         if bg_job.string_stdin is not None:
    994             write_list.append(bg_job.sp.stdin)
    995             reverse_dict[bg_job.sp.stdin] = bg_job
    996 
    997     if timeout:
    998         stop_time = start_time + timeout
    999         time_left = stop_time - time.time()
   1000     else:
   1001         time_left = None # so that select never times out
   1002 
   1003     while not timeout or time_left > 0:
   1004         # select will return when we may write to stdin, when there is
   1005         # stdout/stderr output we can read (including when it is
   1006         # EOF, that is the process has terminated) or when a non-fatal
   1007         # signal was sent to the process. In the last case the select returns
   1008         # EINTR, and we continue waiting for the job if the signal handler for
   1009         # the signal that interrupted the call allows us to.
   1010         try:
   1011             read_ready, write_ready, _ = select.select(read_list, write_list,
   1012                                                        [], SELECT_TIMEOUT)
   1013         except select.error as v:
   1014             if v[0] == errno.EINTR:
   1015                 logging.warning(v)
   1016                 continue
   1017             else:
   1018                 raise
   1019         # os.read() has to be used instead of
   1020         # subproc.stdout.read() which will otherwise block
   1021         for file_obj in read_ready:
   1022             bg_job, is_stdout = reverse_dict[file_obj]
   1023             bg_job.process_output(is_stdout)
   1024 
   1025         for file_obj in write_ready:
   1026             # we can write PIPE_BUF bytes without blocking
   1027             # POSIX requires PIPE_BUF is >= 512
   1028             bg_job = reverse_dict[file_obj]
   1029             file_obj.write(bg_job.string_stdin[:512])
   1030             bg_job.string_stdin = bg_job.string_stdin[512:]
   1031             # no more input data, close stdin, remove it from the select set
   1032             if not bg_job.string_stdin:
   1033                 file_obj.close()
   1034                 write_list.remove(file_obj)
   1035                 del reverse_dict[file_obj]
   1036 
   1037         all_jobs_finished = True
   1038         for bg_job in bg_jobs:
   1039             if bg_job.result.exit_status is not None:
   1040                 continue
   1041 
   1042             bg_job.result.exit_status = bg_job.sp.poll()
   1043             if bg_job.result.exit_status is not None:
   1044                 # process exited, remove its stdout/stdin from the select set
   1045                 bg_job.result.duration = time.time() - start_time
   1046                 read_list.remove(bg_job.sp.stdout)
   1047                 read_list.remove(bg_job.sp.stderr)
   1048                 del reverse_dict[bg_job.sp.stdout]
   1049                 del reverse_dict[bg_job.sp.stderr]
   1050             else:
   1051                 all_jobs_finished = False
   1052 
   1053         if all_jobs_finished:
   1054             return False
   1055 
   1056         if timeout:
   1057             time_left = stop_time - time.time()
   1058 
   1059     # Kill all processes which did not complete prior to timeout
   1060     for bg_job in bg_jobs:
   1061         if bg_job.result.exit_status is not None:
   1062             continue
   1063 
   1064         logging.warning('run process timeout (%s) fired on: %s', timeout,
   1065                         bg_job.command)
   1066         if nuke_subprocess(bg_job.sp) is None:
   1067             # If process could not be SIGKILL'd, log kernel stack.
   1068             logging.warning(read_file('/proc/%d/stack' % bg_job.sp.pid))
   1069         bg_job.result.exit_status = bg_job.sp.poll()
   1070         bg_job.result.duration = time.time() - start_time
   1071 
   1072     return True
   1073 
   1074 
   1075 def pid_is_alive(pid):
   1076     """
   1077     True if process pid exists and is not yet stuck in Zombie state.
   1078     Zombies are impossible to move between cgroups, etc.
   1079     pid can be integer, or text of integer.
   1080     """
   1081     path = '/proc/%s/stat' % pid
   1082 
   1083     try:
   1084         stat = read_one_line(path)
   1085     except IOError:
   1086         if not os.path.exists(path):
   1087             # file went away
   1088             return False
   1089         raise
   1090 
   1091     return stat.split()[2] != 'Z'
   1092 
   1093 
   1094 def signal_pid(pid, sig):
   1095     """
   1096     Sends a signal to a process id. Returns True if the process terminated
   1097     successfully, False otherwise.
   1098     """
   1099     try:
   1100         os.kill(pid, sig)
   1101     except OSError:
   1102         # The process may have died before we could kill it.
   1103         pass
   1104 
   1105     for i in range(5):
   1106         if not pid_is_alive(pid):
   1107             return True
   1108         time.sleep(1)
   1109 
   1110     # The process is still alive
   1111     return False
   1112 
   1113 
   1114 def nuke_subprocess(subproc):
   1115     # check if the subprocess is still alive, first
   1116     if subproc.poll() is not None:
   1117         return subproc.poll()
   1118 
   1119     # the process has not terminated within timeout,
   1120     # kill it via an escalating series of signals.
   1121     signal_queue = [signal.SIGTERM, signal.SIGKILL]
   1122     for sig in signal_queue:
   1123         signal_pid(subproc.pid, sig)
   1124         if subproc.poll() is not None:
   1125             return subproc.poll()
   1126 
   1127 
   1128 def nuke_pid(pid, signal_queue=(signal.SIGTERM, signal.SIGKILL)):
   1129     # the process has not terminated within timeout,
   1130     # kill it via an escalating series of signals.
   1131     pid_path = '/proc/%d/'
   1132     if not os.path.exists(pid_path % pid):
   1133         # Assume that if the pid does not exist in proc it is already dead.
   1134         logging.error('No listing in /proc for pid:%d.', pid)
   1135         raise error.AutoservPidAlreadyDeadError('Could not kill nonexistant '
   1136                                                 'pid: %s.', pid)
   1137     for sig in signal_queue:
   1138         if signal_pid(pid, sig):
   1139             return
   1140 
   1141     # no signal successfully terminated the process
   1142     raise error.AutoservRunError('Could not kill %d for process name: %s' % (
   1143             pid, get_process_name(pid)), None)
   1144 
   1145 
   1146 def system(command, timeout=None, ignore_status=False):
   1147     """
   1148     Run a command
   1149 
   1150     @param timeout: timeout in seconds
   1151     @param ignore_status: if ignore_status=False, throw an exception if the
   1152             command's exit code is non-zero
   1153             if ignore_stauts=True, return the exit code.
   1154 
   1155     @return exit status of command
   1156             (note, this will always be zero unless ignore_status=True)
   1157     """
   1158     return run(command, timeout=timeout, ignore_status=ignore_status,
   1159                stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS).exit_status
   1160 
   1161 
   1162 def system_parallel(commands, timeout=None, ignore_status=False):
   1163     """This function returns a list of exit statuses for the respective
   1164     list of commands."""
   1165     return [bg_jobs.exit_status for bg_jobs in
   1166             run_parallel(commands, timeout=timeout, ignore_status=ignore_status,
   1167                          stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)]
   1168 
   1169 
   1170 def system_output(command, timeout=None, ignore_status=False,
   1171                   retain_output=False, args=()):
   1172     """
   1173     Run a command and return the stdout output.
   1174 
   1175     @param command: command string to execute.
   1176     @param timeout: time limit in seconds before attempting to kill the
   1177             running process. The function will take a few seconds longer
   1178             than 'timeout' to complete if it has to kill the process.
   1179     @param ignore_status: do not raise an exception, no matter what the exit
   1180             code of the command is.
   1181     @param retain_output: set to True to make stdout/stderr of the command
   1182             output to be also sent to the logging system
   1183     @param args: sequence of strings of arguments to be given to the command
   1184             inside " quotes after they have been escaped for that; each
   1185             element in the sequence will be given as a separate command
   1186             argument
   1187 
   1188     @return a string with the stdout output of the command.
   1189     """
   1190     if retain_output:
   1191         out = run(command, timeout=timeout, ignore_status=ignore_status,
   1192                   stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS,
   1193                   args=args).stdout
   1194     else:
   1195         out = run(command, timeout=timeout, ignore_status=ignore_status,
   1196                   args=args).stdout
   1197     if out[-1:] == '\n':
   1198         out = out[:-1]
   1199     return out
   1200 
   1201 
   1202 def system_output_parallel(commands, timeout=None, ignore_status=False,
   1203                            retain_output=False):
   1204     if retain_output:
   1205         out = [bg_job.stdout for bg_job
   1206                in run_parallel(commands, timeout=timeout,
   1207                                ignore_status=ignore_status,
   1208                                stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)]
   1209     else:
   1210         out = [bg_job.stdout for bg_job in run_parallel(commands,
   1211                                   timeout=timeout, ignore_status=ignore_status)]
   1212     for x in out:
   1213         if out[-1:] == '\n': out = out[:-1]
   1214     return out
   1215 
   1216 
   1217 def strip_unicode(input):
   1218     if type(input) == list:
   1219         return [strip_unicode(i) for i in input]
   1220     elif type(input) == dict:
   1221         output = {}
   1222         for key in input.keys():
   1223             output[str(key)] = strip_unicode(input[key])
   1224         return output
   1225     elif type(input) == unicode:
   1226         return str(input)
   1227     else:
   1228         return input
   1229 
   1230 
   1231 def get_cpu_percentage(function, *args, **dargs):
   1232     """Returns a tuple containing the CPU% and return value from function call.
   1233 
   1234     This function calculates the usage time by taking the difference of
   1235     the user and system times both before and after the function call.
   1236     """
   1237     child_pre = resource.getrusage(resource.RUSAGE_CHILDREN)
   1238     self_pre = resource.getrusage(resource.RUSAGE_SELF)
   1239     start = time.time()
   1240     to_return = function(*args, **dargs)
   1241     elapsed = time.time() - start
   1242     self_post = resource.getrusage(resource.RUSAGE_SELF)
   1243     child_post = resource.getrusage(resource.RUSAGE_CHILDREN)
   1244 
   1245     # Calculate CPU Percentage
   1246     s_user, s_system = [a - b for a, b in zip(self_post, self_pre)[:2]]
   1247     c_user, c_system = [a - b for a, b in zip(child_post, child_pre)[:2]]
   1248     cpu_percent = (s_user + c_user + s_system + c_system) / elapsed
   1249 
   1250     return cpu_percent, to_return
   1251 
   1252 
   1253 class SystemLoad(object):
   1254     """
   1255     Get system and/or process values and return average value of load.
   1256     """
   1257     def __init__(self, pids, advanced=False, time_step=0.1, cpu_cont=False,
   1258                  use_log=False):
   1259         """
   1260         @param pids: List of pids to be monitored. If pid = 0 whole system will
   1261           be monitored. pid == 0 means whole system.
   1262         @param advanced: monitor add value for system irq count and softirq
   1263           for process minor and maior page fault
   1264         @param time_step: Time step for continuous monitoring.
   1265         @param cpu_cont: If True monitor CPU load continuously.
   1266         @param use_log: If true every monitoring is logged for dump.
   1267         """
   1268         self.pids = []
   1269         self.stats = {}
   1270         for pid in pids:
   1271             if pid == 0:
   1272                 cpu = FileFieldMonitor("/proc/stat",
   1273                                        [("cpu", 0), # User Time
   1274                                         ("cpu", 2), # System Time
   1275                                         ("intr", 0), # IRQ Count
   1276                                         ("softirq", 0)], # Soft IRQ Count
   1277                                        True,
   1278                                        cpu_cont,
   1279                                        use_log,
   1280                                        " +",
   1281                                        time_step)
   1282                 mem = FileFieldMonitor("/proc/meminfo",
   1283                                        [("MemTotal:", 0), # Mem Total
   1284                                         ("MemFree:", 0), # Mem Free
   1285                                         ("Buffers:", 0), # Buffers
   1286                                         ("Cached:", 0)], # Cached
   1287                                        False,
   1288                                        True,
   1289                                        use_log,
   1290                                        " +",
   1291                                        time_step)
   1292                 self.stats[pid] = ["TOTAL", cpu, mem]
   1293                 self.pids.append(pid)
   1294             else:
   1295                 name = ""
   1296                 if (type(pid) is int):
   1297                     self.pids.append(pid)
   1298                     name = get_process_name(pid)
   1299                 else:
   1300                     self.pids.append(pid[0])
   1301                     name = pid[1]
   1302 
   1303                 cpu = FileFieldMonitor("/proc/%d/stat" %
   1304                                        self.pids[-1],
   1305                                        [("", 13), # User Time
   1306                                         ("", 14), # System Time
   1307                                         ("", 9), # Minority Page Fault
   1308                                         ("", 11)], # Majority Page Fault
   1309                                        True,
   1310                                        cpu_cont,
   1311                                        use_log,
   1312                                        " +",
   1313                                        time_step)
   1314                 mem = FileFieldMonitor("/proc/%d/status" %
   1315                                        self.pids[-1],
   1316                                        [("VmSize:", 0), # Virtual Memory Size
   1317                                         ("VmRSS:", 0), # Resident Set Size
   1318                                         ("VmPeak:", 0), # Peak VM Size
   1319                                         ("VmSwap:", 0)], # VM in Swap
   1320                                        False,
   1321                                        True,
   1322                                        use_log,
   1323                                        " +",
   1324                                        time_step)
   1325                 self.stats[self.pids[-1]] = [name, cpu, mem]
   1326 
   1327         self.advanced = advanced
   1328 
   1329 
   1330     def __str__(self):
   1331         """
   1332         Define format how to print
   1333         """
   1334         out = ""
   1335         for pid in self.pids:
   1336             for stat in self.stats[pid][1:]:
   1337                 out += str(stat.get_status()) + "\n"
   1338         return out
   1339 
   1340 
   1341     def start(self, pids=[]):
   1342         """
   1343         Start monitoring of the process system usage.
   1344         @param pids: List of PIDs you intend to control. Use pids=[] to control
   1345             all defined PIDs.
   1346         """
   1347         if pids == []:
   1348             pids = self.pids
   1349 
   1350         for pid in pids:
   1351             for stat in self.stats[pid][1:]:
   1352                 stat.start()
   1353 
   1354 
   1355     def stop(self, pids=[]):
   1356         """
   1357         Stop monitoring of the process system usage.
   1358         @param pids: List of PIDs you intend to control. Use pids=[] to control
   1359             all defined PIDs.
   1360         """
   1361         if pids == []:
   1362             pids = self.pids
   1363 
   1364         for pid in pids:
   1365             for stat in self.stats[pid][1:]:
   1366                 stat.stop()
   1367 
   1368 
   1369     def dump(self, pids=[]):
   1370         """
   1371         Get the status of monitoring.
   1372         @param pids: List of PIDs you intend to control. Use pids=[] to control
   1373             all defined PIDs.
   1374          @return:
   1375             tuple([cpu load], [memory load]):
   1376                 ([(PID1, (PID1_cpu_meas)), (PID2, (PID2_cpu_meas)), ...],
   1377                  [(PID1, (PID1_mem_meas)), (PID2, (PID2_mem_meas)), ...])
   1378 
   1379             PID1_cpu_meas:
   1380                 average_values[], test_time, cont_meas_values[[]], time_step
   1381             PID1_mem_meas:
   1382                 average_values[], test_time, cont_meas_values[[]], time_step
   1383             where average_values[] are the measured values (mem_free,swap,...)
   1384             which are described in SystemLoad.__init__()-FileFieldMonitor.
   1385             cont_meas_values[[]] is a list of average_values in the sampling
   1386             times.
   1387         """
   1388         if pids == []:
   1389             pids = self.pids
   1390 
   1391         cpus = []
   1392         memory = []
   1393         for pid in pids:
   1394             stat = (pid, self.stats[pid][1].get_status())
   1395             cpus.append(stat)
   1396         for pid in pids:
   1397             stat = (pid, self.stats[pid][2].get_status())
   1398             memory.append(stat)
   1399 
   1400         return (cpus, memory)
   1401 
   1402 
   1403     def get_cpu_status_string(self, pids=[]):
   1404         """
   1405         Convert status to string array.
   1406         @param pids: List of PIDs you intend to control. Use pids=[] to control
   1407             all defined PIDs.
   1408         @return: String format to table.
   1409         """
   1410         if pids == []:
   1411             pids = self.pids
   1412 
   1413         headers = ["NAME",
   1414                    ("%7s") % "PID",
   1415                    ("%5s") % "USER",
   1416                    ("%5s") % "SYS",
   1417                    ("%5s") % "SUM"]
   1418         if self.advanced:
   1419             headers.extend(["MINFLT/IRQC",
   1420                             "MAJFLT/SOFTIRQ"])
   1421         headers.append(("%11s") % "TIME")
   1422         textstatus = []
   1423         for pid in pids:
   1424             stat = self.stats[pid][1].get_status()
   1425             time = stat[1]
   1426             stat = stat[0]
   1427             textstatus.append(["%s" % self.stats[pid][0],
   1428                                "%7s" % pid,
   1429                                "%4.0f%%" % (stat[0] / time),
   1430                                "%4.0f%%" % (stat[1] / time),
   1431                                "%4.0f%%" % ((stat[0] + stat[1]) / time),
   1432                                "%10.3fs" % time])
   1433             if self.advanced:
   1434                 textstatus[-1].insert(-1, "%11d" % stat[2])
   1435                 textstatus[-1].insert(-1, "%14d" % stat[3])
   1436 
   1437         return matrix_to_string(textstatus, tuple(headers))
   1438 
   1439 
   1440     def get_mem_status_string(self, pids=[]):
   1441         """
   1442         Convert status to string array.
   1443         @param pids: List of PIDs you intend to control. Use pids=[] to control
   1444             all defined PIDs.
   1445         @return: String format to table.
   1446         """
   1447         if pids == []:
   1448             pids = self.pids
   1449 
   1450         headers = ["NAME",
   1451                    ("%7s") % "PID",
   1452                    ("%8s") % "TOTAL/VMSIZE",
   1453                    ("%8s") % "FREE/VMRSS",
   1454                    ("%8s") % "BUFFERS/VMPEAK",
   1455                    ("%8s") % "CACHED/VMSWAP",
   1456                    ("%11s") % "TIME"]
   1457         textstatus = []
   1458         for pid in pids:
   1459             stat = self.stats[pid][2].get_status()
   1460             time = stat[1]
   1461             stat = stat[0]
   1462             textstatus.append(["%s" % self.stats[pid][0],
   1463                                "%7s" % pid,
   1464                                "%10dMB" % (stat[0] / 1024),
   1465                                "%8dMB" % (stat[1] / 1024),
   1466                                "%12dMB" % (stat[2] / 1024),
   1467                                "%11dMB" % (stat[3] / 1024),
   1468                                "%10.3fs" % time])
   1469 
   1470         return matrix_to_string(textstatus, tuple(headers))
   1471 
   1472 
   1473 def get_arch(run_function=run):
   1474     """
   1475     Get the hardware architecture of the machine.
   1476     If specified, run_function should return a CmdResult object and throw a
   1477     CmdError exception.
   1478     If run_function is anything other than utils.run(), it is used to
   1479     execute the commands. By default (when set to utils.run()) this will
   1480     just examine os.uname()[4].
   1481     """
   1482 
   1483     # Short circuit from the common case.
   1484     if run_function == run:
   1485         return re.sub(r'i\d86$', 'i386', os.uname()[4])
   1486 
   1487     # Otherwise, use the run_function in case it hits a remote machine.
   1488     arch = run_function('/bin/uname -m').stdout.rstrip()
   1489     if re.match(r'i\d86$', arch):
   1490         arch = 'i386'
   1491     return arch
   1492 
   1493 def get_arch_userspace(run_function=run):
   1494     """
   1495     Get the architecture by userspace (possibly different from kernel).
   1496     """
   1497     archs = {
   1498         'arm': 'ELF 32-bit.*, ARM,',
   1499         'i386': 'ELF 32-bit.*, Intel 80386,',
   1500         'x86_64': 'ELF 64-bit.*, x86-64,',
   1501     }
   1502 
   1503     cmd = 'file --brief --dereference /bin/sh'
   1504     filestr = run_function(cmd).stdout.rstrip()
   1505     for a, regex in archs.iteritems():
   1506         if re.match(regex, filestr):
   1507             return a
   1508 
   1509     return get_arch()
   1510 
   1511 
   1512 def get_num_logical_cpus_per_socket(run_function=run):
   1513     """
   1514     Get the number of cores (including hyperthreading) per cpu.
   1515     run_function is used to execute the commands. It defaults to
   1516     utils.run() but a custom method (if provided) should be of the
   1517     same schema as utils.run. It should return a CmdResult object and
   1518     throw a CmdError exception.
   1519     """
   1520     siblings = run_function('grep "^siblings" /proc/cpuinfo').stdout.rstrip()
   1521     num_siblings = map(int,
   1522                        re.findall(r'^siblings\s*:\s*(\d+)\s*$',
   1523                                   siblings, re.M))
   1524     if len(num_siblings) == 0:
   1525         raise error.TestError('Unable to find siblings info in /proc/cpuinfo')
   1526     if min(num_siblings) != max(num_siblings):
   1527         raise error.TestError('Number of siblings differ %r' %
   1528                               num_siblings)
   1529     return num_siblings[0]
   1530 
   1531 
   1532 def merge_trees(src, dest):
   1533     """
   1534     Merges a source directory tree at 'src' into a destination tree at
   1535     'dest'. If a path is a file in both trees than the file in the source
   1536     tree is APPENDED to the one in the destination tree. If a path is
   1537     a directory in both trees then the directories are recursively merged
   1538     with this function. In any other case, the function will skip the
   1539     paths that cannot be merged (instead of failing).
   1540     """
   1541     if not os.path.exists(src):
   1542         return # exists only in dest
   1543     elif not os.path.exists(dest):
   1544         if os.path.isfile(src):
   1545             shutil.copy2(src, dest) # file only in src
   1546         else:
   1547             shutil.copytree(src, dest, symlinks=True) # dir only in src
   1548         return
   1549     elif os.path.isfile(src) and os.path.isfile(dest):
   1550         # src & dest are files in both trees, append src to dest
   1551         destfile = open(dest, "a")
   1552         try:
   1553             srcfile = open(src)
   1554             try:
   1555                 destfile.write(srcfile.read())
   1556             finally:
   1557                 srcfile.close()
   1558         finally:
   1559             destfile.close()
   1560     elif os.path.isdir(src) and os.path.isdir(dest):
   1561         # src & dest are directories in both trees, so recursively merge
   1562         for name in os.listdir(src):
   1563             merge_trees(os.path.join(src, name), os.path.join(dest, name))
   1564     else:
   1565         # src & dest both exist, but are incompatible
   1566         return
   1567 
   1568 
   1569 class CmdResult(object):
   1570     """
   1571     Command execution result.
   1572 
   1573     command:     String containing the command line itself
   1574     exit_status: Integer exit code of the process
   1575     stdout:      String containing stdout of the process
   1576     stderr:      String containing stderr of the process
   1577     duration:    Elapsed wall clock time running the process
   1578     """
   1579 
   1580 
   1581     def __init__(self, command="", stdout="", stderr="",
   1582                  exit_status=None, duration=0):
   1583         self.command = command
   1584         self.exit_status = exit_status
   1585         self.stdout = stdout
   1586         self.stderr = stderr
   1587         self.duration = duration
   1588 
   1589 
   1590     def __repr__(self):
   1591         wrapper = textwrap.TextWrapper(width = 78,
   1592                                        initial_indent="\n    ",
   1593                                        subsequent_indent="    ")
   1594 
   1595         stdout = self.stdout.rstrip()
   1596         if stdout:
   1597             stdout = "\nstdout:\n%s" % stdout
   1598 
   1599         stderr = self.stderr.rstrip()
   1600         if stderr:
   1601             stderr = "\nstderr:\n%s" % stderr
   1602 
   1603         return ("* Command: %s\n"
   1604                 "Exit status: %s\n"
   1605                 "Duration: %s\n"
   1606                 "%s"
   1607                 "%s"
   1608                 % (wrapper.fill(str(self.command)), self.exit_status,
   1609                 self.duration, stdout, stderr))
   1610 
   1611 
   1612 class run_randomly:
   1613     def __init__(self, run_sequentially=False):
   1614         # Run sequentially is for debugging control files
   1615         self.test_list = []
   1616         self.run_sequentially = run_sequentially
   1617 
   1618 
   1619     def add(self, *args, **dargs):
   1620         test = (args, dargs)
   1621         self.test_list.append(test)
   1622 
   1623 
   1624     def run(self, fn):
   1625         while self.test_list:
   1626             test_index = random.randint(0, len(self.test_list)-1)
   1627             if self.run_sequentially:
   1628                 test_index = 0
   1629             (args, dargs) = self.test_list.pop(test_index)
   1630             fn(*args, **dargs)
   1631 
   1632 
   1633 def import_site_module(path, module, dummy=None, modulefile=None):
   1634     """
   1635     Try to import the site specific module if it exists.
   1636 
   1637     @param path full filename of the source file calling this (ie __file__)
   1638     @param module full module name
   1639     @param dummy dummy value to return in case there is no symbol to import
   1640     @param modulefile module filename
   1641 
   1642     @return site specific module or dummy
   1643 
   1644     @raises ImportError if the site file exists but imports fails
   1645     """
   1646     short_module = module[module.rfind(".") + 1:]
   1647 
   1648     if not modulefile:
   1649         modulefile = short_module + ".py"
   1650 
   1651     if os.path.exists(os.path.join(os.path.dirname(path), modulefile)):
   1652         return __import__(module, {}, {}, [short_module])
   1653     return dummy
   1654 
   1655 
   1656 def import_site_symbol(path, module, name, dummy=None, modulefile=None):
   1657     """
   1658     Try to import site specific symbol from site specific file if it exists
   1659 
   1660     @param path full filename of the source file calling this (ie __file__)
   1661     @param module full module name
   1662     @param name symbol name to be imported from the site file
   1663     @param dummy dummy value to return in case there is no symbol to import
   1664     @param modulefile module filename
   1665 
   1666     @return site specific symbol or dummy
   1667 
   1668     @raises ImportError if the site file exists but imports fails
   1669     """
   1670     module = import_site_module(path, module, modulefile=modulefile)
   1671     if not module:
   1672         return dummy
   1673 
   1674     # special unique value to tell us if the symbol can't be imported
   1675     cant_import = object()
   1676 
   1677     obj = getattr(module, name, cant_import)
   1678     if obj is cant_import:
   1679         return dummy
   1680 
   1681     return obj
   1682 
   1683 
   1684 def import_site_class(path, module, classname, baseclass, modulefile=None):
   1685     """
   1686     Try to import site specific class from site specific file if it exists
   1687 
   1688     Args:
   1689         path: full filename of the source file calling this (ie __file__)
   1690         module: full module name
   1691         classname: class name to be loaded from site file
   1692         baseclass: base class object to return when no site file present or
   1693             to mixin when site class exists but is not inherited from baseclass
   1694         modulefile: module filename
   1695 
   1696     Returns: baseclass if site specific class does not exist, the site specific
   1697         class if it exists and is inherited from baseclass or a mixin of the
   1698         site specific class and baseclass when the site specific class exists
   1699         and is not inherited from baseclass
   1700 
   1701     Raises: ImportError if the site file exists but imports fails
   1702     """
   1703 
   1704     res = import_site_symbol(path, module, classname, None, modulefile)
   1705     if res:
   1706         if not issubclass(res, baseclass):
   1707             # if not a subclass of baseclass then mix in baseclass with the
   1708             # site specific class object and return the result
   1709             res = type(classname, (res, baseclass), {})
   1710     else:
   1711         res = baseclass
   1712 
   1713     return res
   1714 
   1715 
   1716 def import_site_function(path, module, funcname, dummy, modulefile=None):
   1717     """
   1718     Try to import site specific function from site specific file if it exists
   1719 
   1720     Args:
   1721         path: full filename of the source file calling this (ie __file__)
   1722         module: full module name
   1723         funcname: function name to be imported from site file
   1724         dummy: dummy function to return in case there is no function to import
   1725         modulefile: module filename
   1726 
   1727     Returns: site specific function object or dummy
   1728 
   1729     Raises: ImportError if the site file exists but imports fails
   1730     """
   1731 
   1732     return import_site_symbol(path, module, funcname, dummy, modulefile)
   1733 
   1734 
   1735 def _get_pid_path(program_name):
   1736     my_path = os.path.dirname(__file__)
   1737     return os.path.abspath(os.path.join(my_path, "..", "..",
   1738                                         "%s.pid" % program_name))
   1739 
   1740 
   1741 def write_pid(program_name):
   1742     """
   1743     Try to drop <program_name>.pid in the main autotest directory.
   1744 
   1745     Args:
   1746       program_name: prefix for file name
   1747     """
   1748     pidfile = open(_get_pid_path(program_name), "w")
   1749     try:
   1750         pidfile.write("%s\n" % os.getpid())
   1751     finally:
   1752         pidfile.close()
   1753 
   1754 
   1755 def delete_pid_file_if_exists(program_name):
   1756     """
   1757     Tries to remove <program_name>.pid from the main autotest directory.
   1758     """
   1759     pidfile_path = _get_pid_path(program_name)
   1760 
   1761     try:
   1762         os.remove(pidfile_path)
   1763     except OSError:
   1764         if not os.path.exists(pidfile_path):
   1765             return
   1766         raise
   1767 
   1768 
   1769 def get_pid_from_file(program_name):
   1770     """
   1771     Reads the pid from <program_name>.pid in the autotest directory.
   1772 
   1773     @param program_name the name of the program
   1774     @return the pid if the file exists, None otherwise.
   1775     """
   1776     pidfile_path = _get_pid_path(program_name)
   1777     if not os.path.exists(pidfile_path):
   1778         return None
   1779 
   1780     pidfile = open(_get_pid_path(program_name), 'r')
   1781 
   1782     try:
   1783         try:
   1784             pid = int(pidfile.readline())
   1785         except IOError:
   1786             if not os.path.exists(pidfile_path):
   1787                 return None
   1788             raise
   1789     finally:
   1790         pidfile.close()
   1791 
   1792     return pid
   1793 
   1794 
   1795 def get_process_name(pid):
   1796     """
   1797     Get process name from PID.
   1798     @param pid: PID of process.
   1799     @return: Process name if PID stat file exists or 'Dead PID' if it does not.
   1800     """
   1801     pid_stat_path = "/proc/%d/stat"
   1802     if not os.path.exists(pid_stat_path % pid):
   1803         return "Dead Pid"
   1804     return get_field(read_file(pid_stat_path % pid), 1)[1:-1]
   1805 
   1806 
   1807 def program_is_alive(program_name):
   1808     """
   1809     Checks if the process is alive and not in Zombie state.
   1810 
   1811     @param program_name the name of the program
   1812     @return True if still alive, False otherwise
   1813     """
   1814     pid = get_pid_from_file(program_name)
   1815     if pid is None:
   1816         return False
   1817     return pid_is_alive(pid)
   1818 
   1819 
   1820 def signal_program(program_name, sig=signal.SIGTERM):
   1821     """
   1822     Sends a signal to the process listed in <program_name>.pid
   1823 
   1824     @param program_name the name of the program
   1825     @param sig signal to send
   1826     """
   1827     pid = get_pid_from_file(program_name)
   1828     if pid:
   1829         signal_pid(pid, sig)
   1830 
   1831 
   1832 def get_relative_path(path, reference):
   1833     """Given 2 absolute paths "path" and "reference", compute the path of
   1834     "path" as relative to the directory "reference".
   1835 
   1836     @param path the absolute path to convert to a relative path
   1837     @param reference an absolute directory path to which the relative
   1838         path will be computed
   1839     """
   1840     # normalize the paths (remove double slashes, etc)
   1841     assert(os.path.isabs(path))
   1842     assert(os.path.isabs(reference))
   1843 
   1844     path = os.path.normpath(path)
   1845     reference = os.path.normpath(reference)
   1846 
   1847     # we could use os.path.split() but it splits from the end
   1848     path_list = path.split(os.path.sep)[1:]
   1849     ref_list = reference.split(os.path.sep)[1:]
   1850 
   1851     # find the longest leading common path
   1852     for i in xrange(min(len(path_list), len(ref_list))):
   1853         if path_list[i] != ref_list[i]:
   1854             # decrement i so when exiting this loop either by no match or by
   1855             # end of range we are one step behind
   1856             i -= 1
   1857             break
   1858     i += 1
   1859     # drop the common part of the paths, not interested in that anymore
   1860     del path_list[:i]
   1861 
   1862     # for each uncommon component in the reference prepend a ".."
   1863     path_list[:0] = ['..'] * (len(ref_list) - i)
   1864 
   1865     return os.path.join(*path_list)
   1866 
   1867 
   1868 def sh_escape(command):
   1869     """
   1870     Escape special characters from a command so that it can be passed
   1871     as a double quoted (" ") string in a (ba)sh command.
   1872 
   1873     Args:
   1874             command: the command string to escape.
   1875 
   1876     Returns:
   1877             The escaped command string. The required englobing double
   1878             quotes are NOT added and so should be added at some point by
   1879             the caller.
   1880 
   1881     See also: http://www.tldp.org/LDP/abs/html/escapingsection.html
   1882     """
   1883     command = command.replace("\\", "\\\\")
   1884     command = command.replace("$", r'\$')
   1885     command = command.replace('"', r'\"')
   1886     command = command.replace('`', r'\`')
   1887     return command
   1888 
   1889 
   1890 def sh_quote_word(text, whitelist=SHELL_QUOTING_WHITELIST):
   1891     r"""Quote a string to make it safe as a single word in a shell command.
   1892 
   1893     POSIX shell syntax recognizes no escape characters inside a single-quoted
   1894     string.  So, single quotes can safely quote any string of characters except
   1895     a string with a single quote character.  A single quote character must be
   1896     quoted with the sequence '\'' which translates to:
   1897         '  -> close current quote
   1898         \' -> insert a literal single quote
   1899         '  -> reopen quoting again.
   1900 
   1901     This is safe for all combinations of characters, including embedded and
   1902     trailing backslashes in odd or even numbers.
   1903 
   1904     This is also safe for nesting, e.g. the following is a valid use:
   1905 
   1906         adb_command = 'adb shell %s' % (
   1907                 sh_quote_word('echo %s' % sh_quote_word('hello world')))
   1908 
   1909     @param text: The string to be quoted into a single word for the shell.
   1910     @param whitelist: Optional list of characters that do not need quoting.
   1911                       Defaults to a known good list of characters.
   1912 
   1913     @return A string, possibly quoted, safe as a single word for a shell.
   1914     """
   1915     if all(c in whitelist for c in text):
   1916         return text
   1917     return "'" + text.replace("'", r"'\''") + "'"
   1918 
   1919 
   1920 def configure(extra=None, configure='./configure'):
   1921     """
   1922     Run configure passing in the correct host, build, and target options.
   1923 
   1924     @param extra: extra command line arguments to pass to configure
   1925     @param configure: which configure script to use
   1926     """
   1927     args = []
   1928     if 'CHOST' in os.environ:
   1929         args.append('--host=' + os.environ['CHOST'])
   1930     if 'CBUILD' in os.environ:
   1931         args.append('--build=' + os.environ['CBUILD'])
   1932     if 'CTARGET' in os.environ:
   1933         args.append('--target=' + os.environ['CTARGET'])
   1934     if extra:
   1935         args.append(extra)
   1936 
   1937     system('%s %s' % (configure, ' '.join(args)))
   1938 
   1939 
   1940 def make(extra='', make='make', timeout=None, ignore_status=False):
   1941     """
   1942     Run make, adding MAKEOPTS to the list of options.
   1943 
   1944     @param extra: extra command line arguments to pass to make.
   1945     """
   1946     cmd = '%s %s %s' % (make, os.environ.get('MAKEOPTS', ''), extra)
   1947     return system(cmd, timeout=timeout, ignore_status=ignore_status)
   1948 
   1949 
   1950 def compare_versions(ver1, ver2):
   1951     """Version number comparison between ver1 and ver2 strings.
   1952 
   1953     >>> compare_tuple("1", "2")
   1954     -1
   1955     >>> compare_tuple("foo-1.1", "foo-1.2")
   1956     -1
   1957     >>> compare_tuple("1.2", "1.2a")
   1958     -1
   1959     >>> compare_tuple("1.2b", "1.2a")
   1960     1
   1961     >>> compare_tuple("1.3.5.3a", "1.3.5.3b")
   1962     -1
   1963 
   1964     Args:
   1965         ver1: version string
   1966         ver2: version string
   1967 
   1968     Returns:
   1969         int:  1 if ver1 >  ver2
   1970               0 if ver1 == ver2
   1971              -1 if ver1 <  ver2
   1972     """
   1973     ax = re.split('[.-]', ver1)
   1974     ay = re.split('[.-]', ver2)
   1975     while len(ax) > 0 and len(ay) > 0:
   1976         cx = ax.pop(0)
   1977         cy = ay.pop(0)
   1978         maxlen = max(len(cx), len(cy))
   1979         c = cmp(cx.zfill(maxlen), cy.zfill(maxlen))
   1980         if c != 0:
   1981             return c
   1982     return cmp(len(ax), len(ay))
   1983 
   1984 
   1985 def args_to_dict(args):
   1986     """Convert autoserv extra arguments in the form of key=val or key:val to a
   1987     dictionary.  Each argument key is converted to lowercase dictionary key.
   1988 
   1989     Args:
   1990         args - list of autoserv extra arguments.
   1991 
   1992     Returns:
   1993         dictionary
   1994     """
   1995     arg_re = re.compile(r'(\w+)[:=](.*)$')
   1996     dict = {}
   1997     for arg in args:
   1998         match = arg_re.match(arg)
   1999         if match:
   2000             dict[match.group(1).lower()] = match.group(2)
   2001         else:
   2002             logging.warning("args_to_dict: argument '%s' doesn't match "
   2003                             "'%s' pattern. Ignored.", arg, arg_re.pattern)
   2004     return dict
   2005 
   2006 
   2007 def get_unused_port():
   2008     """
   2009     Finds a semi-random available port. A race condition is still
   2010     possible after the port number is returned, if another process
   2011     happens to bind it.
   2012 
   2013     Returns:
   2014         A port number that is unused on both TCP and UDP.
   2015     """
   2016 
   2017     def try_bind(port, socket_type, socket_proto):
   2018         s = socket.socket(socket.AF_INET, socket_type, socket_proto)
   2019         try:
   2020             try:
   2021                 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
   2022                 s.bind(('', port))
   2023                 return s.getsockname()[1]
   2024             except socket.error:
   2025                 return None
   2026         finally:
   2027             s.close()
   2028 
   2029     # On the 2.6 kernel, calling try_bind() on UDP socket returns the
   2030     # same port over and over. So always try TCP first.
   2031     while True:
   2032         # Ask the OS for an unused port.
   2033         port = try_bind(0, socket.SOCK_STREAM, socket.IPPROTO_TCP)
   2034         # Check if this port is unused on the other protocol.
   2035         if port and try_bind(port, socket.SOCK_DGRAM, socket.IPPROTO_UDP):
   2036             return port
   2037 
   2038 
   2039 def ask(question, auto=False):
   2040     """
   2041     Raw input with a prompt that emulates logging.
   2042 
   2043     @param question: Question to be asked
   2044     @param auto: Whether to return "y" instead of asking the question
   2045     """
   2046     if auto:
   2047         logging.info("%s (y/n) y", question)
   2048         return "y"
   2049     return raw_input("%s INFO | %s (y/n) " %
   2050                      (time.strftime("%H:%M:%S", time.localtime()), question))
   2051 
   2052 
   2053 def rdmsr(address, cpu=0):
   2054     """
   2055     Reads an x86 MSR from the specified CPU, returns as long integer.
   2056     """
   2057     with open('/dev/cpu/%s/msr' % cpu, 'r', 0) as fd:
   2058         fd.seek(address)
   2059         return struct.unpack('=Q', fd.read(8))[0]
   2060 
   2061 
   2062 def wait_for_value(func,
   2063                    expected_value=None,
   2064                    min_threshold=None,
   2065                    max_threshold=None,
   2066                    timeout_sec=10):
   2067     """
   2068     Returns the value of func().  If |expected_value|, |min_threshold|, and
   2069     |max_threshold| are not set, returns immediately.
   2070 
   2071     If |expected_value| is set, polls the return value until |expected_value| is
   2072     reached, and returns that value.
   2073 
   2074     If either |max_threshold| or |min_threshold| is set, this function will
   2075     will repeatedly call func() until the return value reaches or exceeds one of
   2076     these thresholds.
   2077 
   2078     Polling will stop after |timeout_sec| regardless of these thresholds.
   2079 
   2080     @param func: function whose return value is to be waited on.
   2081     @param expected_value: wait for func to return this value.
   2082     @param min_threshold: wait for func value to reach or fall below this value.
   2083     @param max_threshold: wait for func value to reach or rise above this value.
   2084     @param timeout_sec: Number of seconds to wait before giving up and
   2085                         returning whatever value func() last returned.
   2086 
   2087     Return value:
   2088         The most recent return value of func().
   2089     """
   2090     value = None
   2091     start_time_sec = time.time()
   2092     while True:
   2093         value = func()
   2094         if (expected_value is None and \
   2095             min_threshold is None and \
   2096             max_threshold is None) or \
   2097            (expected_value is not None and value == expected_value) or \
   2098            (min_threshold is not None and value <= min_threshold) or \
   2099            (max_threshold is not None and value >= max_threshold):
   2100             break
   2101 
   2102         if time.time() - start_time_sec >= timeout_sec:
   2103             break
   2104         time.sleep(0.1)
   2105 
   2106     return value
   2107 
   2108 
   2109 def wait_for_value_changed(func,
   2110                            old_value=None,
   2111                            timeout_sec=10):
   2112     """
   2113     Returns the value of func().
   2114 
   2115     The function polls the return value until it is different from |old_value|,
   2116     and returns that value.
   2117 
   2118     Polling will stop after |timeout_sec|.
   2119 
   2120     @param func: function whose return value is to be waited on.
   2121     @param old_value: wait for func to return a value different from this.
   2122     @param timeout_sec: Number of seconds to wait before giving up and
   2123                         returning whatever value func() last returned.
   2124 
   2125     @returns The most recent return value of func().
   2126     """
   2127     value = None
   2128     start_time_sec = time.time()
   2129     while True:
   2130         value = func()
   2131         if value != old_value:
   2132             break
   2133 
   2134         if time.time() - start_time_sec >= timeout_sec:
   2135             break
   2136         time.sleep(0.1)
   2137 
   2138     return value
   2139 
   2140 
   2141 def restart_job(name):
   2142     """
   2143     Restarts an upstart job if it's running.
   2144     If it's not running, start it.
   2145     """
   2146 
   2147     if system_output('status %s' % name).find('start/running') != -1:
   2148         system_output('restart %s' % name)
   2149     else:
   2150         system_output('start %s' % name)
   2151 
   2152