Home | History | Annotate | Download | only in lit 
      1 import errno
      2 import itertools
      3 import math
      4 import os
      5 import platform
      6 import signal
      7 import subprocess
      8 import sys
      9 import threading
     10 
     11 def to_bytes(str):
     12     # Encode to UTF-8 to get binary data.
     13     return str.encode('utf-8')
     14 
     15 def to_string(bytes):
     16     if isinstance(bytes, str):
     17         return bytes
     18     return to_bytes(bytes)
     19 
     20 def convert_string(bytes):
     21     try:
     22         return to_string(bytes.decode('utf-8'))
     23     except UnicodeError:
     24         return str(bytes)
     25 
     26 def detectCPUs():
     27     """
     28     Detects the number of CPUs on a system. Cribbed from pp.
     29     """
     30     # Linux, Unix and MacOS:
     31     if hasattr(os, "sysconf"):
     32         if "SC_NPROCESSORS_ONLN" in os.sysconf_names:
     33             # Linux & Unix:
     34             ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
     35             if isinstance(ncpus, int) and ncpus > 0:
     36                 return ncpus
     37         else: # OSX:
     38             return int(capture(['sysctl', '-n', 'hw.ncpu']))
     39     # Windows:
     40     if "NUMBER_OF_PROCESSORS" in os.environ:
     41         ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
     42         if ncpus > 0:
     43             # With more than 32 processes, process creation often fails with
     44             # "Too many open files".  FIXME: Check if there's a better fix.
     45             return min(ncpus, 32)
     46     return 1 # Default
     47 
     48 def mkdir_p(path):
     49     """mkdir_p(path) - Make the "path" directory, if it does not exist; this
     50     will also make directories for any missing parent directories."""
     51     if not path or os.path.exists(path):
     52         return
     53 
     54     parent = os.path.dirname(path)
     55     if parent != path:
     56         mkdir_p(parent)
     57 
     58     try:
     59         os.mkdir(path)
     60     except OSError:
     61         e = sys.exc_info()[1]
     62         # Ignore EEXIST, which may occur during a race condition.
     63         if e.errno != errno.EEXIST:
     64             raise
     65 
     66 def capture(args, env=None):
     67     """capture(command) - Run the given command (or argv list) in a shell and
     68     return the standard output."""
     69     p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
     70                          env=env)
     71     out,_ = p.communicate()
     72     return convert_string(out)
     73 
     74 def which(command, paths = None):
     75     """which(command, [paths]) - Look up the given command in the paths string
     76     (or the PATH environment variable, if unspecified)."""
     77 
     78     if paths is None:
     79         paths = os.environ.get('PATH','')
     80 
     81     # Check for absolute match first.
     82     if os.path.isfile(command):
     83         return command
     84 
     85     # Would be nice if Python had a lib function for this.
     86     if not paths:
     87         paths = os.defpath
     88 
     89     # Get suffixes to search.
     90     # On Cygwin, 'PATHEXT' may exist but it should not be used.
     91     if os.pathsep == ';':
     92         pathext = os.environ.get('PATHEXT', '').split(';')
     93     else:
     94         pathext = ['']
     95 
     96     # Search the paths...
     97     for path in paths.split(os.pathsep):
     98         for ext in pathext:
     99             p = os.path.join(path, command + ext)
    100             if os.path.exists(p) and not os.path.isdir(p):
    101                 return p
    102 
    103     return None
    104 
    105 def checkToolsPath(dir, tools):
    106     for tool in tools:
    107         if not os.path.exists(os.path.join(dir, tool)):
    108             return False;
    109     return True;
    110 
    111 def whichTools(tools, paths):
    112     for path in paths.split(os.pathsep):
    113         if checkToolsPath(path, tools):
    114             return path
    115     return None
    116 
    117 def printHistogram(items, title = 'Items'):
    118     items.sort(key = lambda item: item[1])
    119 
    120     maxValue = max([v for _,v in items])
    121 
    122     # Select first "nice" bar height that produces more than 10 bars.
    123     power = int(math.ceil(math.log(maxValue, 10)))
    124     for inc in itertools.cycle((5, 2, 2.5, 1)):
    125         barH = inc * 10**power
    126         N = int(math.ceil(maxValue / barH))
    127         if N > 10:
    128             break
    129         elif inc == 1:
    130             power -= 1
    131 
    132     histo = [set() for i in range(N)]
    133     for name,v in items:
    134         bin = min(int(N * v/maxValue), N-1)
    135         histo[bin].add(name)
    136 
    137     barW = 40
    138     hr = '-' * (barW + 34)
    139     print('\nSlowest %s:' % title)
    140     print(hr)
    141     for name,value in items[-20:]:
    142         print('%.2fs: %s' % (value, name))
    143     print('\n%s Times:' % title)
    144     print(hr)
    145     pDigits = int(math.ceil(math.log(maxValue, 10)))
    146     pfDigits = max(0, 3-pDigits)
    147     if pfDigits:
    148         pDigits += pfDigits + 1
    149     cDigits = int(math.ceil(math.log(len(items), 10)))
    150     print("[%s] :: [%s] :: [%s]" % ('Range'.center((pDigits+1)*2 + 3),
    151                                     'Percentage'.center(barW),
    152                                     'Count'.center(cDigits*2 + 1)))
    153     print(hr)
    154     for i,row in enumerate(histo):
    155         pct = float(len(row)) / len(items)
    156         w = int(barW * pct)
    157         print("[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]" % (
    158             pDigits, pfDigits, i*barH, pDigits, pfDigits, (i+1)*barH,
    159             '*'*w, ' '*(barW-w), cDigits, len(row), cDigits, len(items)))
    160 
    161 class ExecuteCommandTimeoutException(Exception):
    162     def __init__(self, msg, out, err, exitCode):
    163         assert isinstance(msg, str)
    164         assert isinstance(out, str)
    165         assert isinstance(err, str)
    166         assert isinstance(exitCode, int)
    167         self.msg = msg
    168         self.out = out
    169         self.err = err
    170         self.exitCode = exitCode
    171 
    172 # Close extra file handles on UNIX (on Windows this cannot be done while
    173 # also redirecting input).
    174 kUseCloseFDs = not (platform.system() == 'Windows')
    175 def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
    176     """
    177         Execute command ``command`` (list of arguments or string)
    178         with
    179         * working directory ``cwd`` (str), use None to use the current
    180           working directory
    181         * environment ``env`` (dict), use None for none
    182         * Input to the command ``input`` (str), use string to pass
    183           no input.
    184         * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
    185 
    186         Returns a tuple (out, err, exitCode) where
    187         * ``out`` (str) is the standard output of running the command
    188         * ``err`` (str) is the standard error of running the command
    189         * ``exitCode`` (int) is the exitCode of running the command
    190 
    191         If the timeout is hit an ``ExecuteCommandTimeoutException``
    192         is raised.
    193     """
    194     p = subprocess.Popen(command, cwd=cwd,
    195                          stdin=subprocess.PIPE,
    196                          stdout=subprocess.PIPE,
    197                          stderr=subprocess.PIPE,
    198                          env=env, close_fds=kUseCloseFDs)
    199     timerObject = None
    200     # FIXME: Because of the way nested function scopes work in Python 2.x we
    201     # need to use a reference to a mutable object rather than a plain
    202     # bool. In Python 3 we could use the "nonlocal" keyword but we need
    203     # to support Python 2 as well.
    204     hitTimeOut = [False]
    205     try:
    206         if timeout > 0:
    207             def killProcess():
    208                 # We may be invoking a shell so we need to kill the
    209                 # process and all its children.
    210                 hitTimeOut[0] = True
    211                 killProcessAndChildren(p.pid)
    212 
    213             timerObject = threading.Timer(timeout, killProcess)
    214             timerObject.start()
    215 
    216         out,err = p.communicate(input=input)
    217         exitCode = p.wait()
    218     finally:
    219         if timerObject != None:
    220             timerObject.cancel()
    221 
    222     # Ensure the resulting output is always of string type.
    223     out = convert_string(out)
    224     err = convert_string(err)
    225 
    226     if hitTimeOut[0]:
    227         raise ExecuteCommandTimeoutException(
    228             msg='Reached timeout of {} seconds'.format(timeout),
    229             out=out,
    230             err=err,
    231             exitCode=exitCode
    232             )
    233 
    234     # Detect Ctrl-C in subprocess.
    235     if exitCode == -signal.SIGINT:
    236         raise KeyboardInterrupt
    237 
    238     return out, err, exitCode
    239 
    240 def usePlatformSdkOnDarwin(config, lit_config):
    241     # On Darwin, support relocatable SDKs by providing Clang with a
    242     # default system root path.
    243     if 'darwin' in config.target_triple:
    244         try:
    245             cmd = subprocess.Popen(['xcrun', '--show-sdk-path'],
    246                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    247             out, err = cmd.communicate()
    248             out = out.strip()
    249             res = cmd.wait()
    250         except OSError:
    251             res = -1
    252         if res == 0 and out:
    253             sdk_path = out
    254             lit_config.note('using SDKROOT: %r' % sdk_path)
    255             config.environment['SDKROOT'] = sdk_path
    256 
    257 def killProcessAndChildren(pid):
    258     """
    259     This function kills a process with ``pid`` and all its
    260     running children (recursively). It is currently implemented
    261     using the psutil module which provides a simple platform
    262     neutral implementation.
    263 
    264     TODO: Reimplement this without using psutil so we can
    265           remove our dependency on it.
    266     """
    267     import psutil
    268     try:
    269         psutilProc = psutil.Process(pid)
    270         # Handle the different psutil API versions
    271         try:
    272             # psutil >= 2.x
    273             children_iterator = psutilProc.children(recursive=True)
    274         except AttributeError:
    275             # psutil 1.x
    276             children_iterator = psutilProc.get_children(recursive=True)
    277         for child in children_iterator:
    278             try:
    279                 child.kill()
    280             except psutil.NoSuchProcess:
    281                 pass
    282         psutilProc.kill()
    283     except psutil.NoSuchProcess:
    284         pass
    285