subzero/pydir/szbuild.py

#!/usr/bin/env python2

import argparse
import os
import pipes
import re
import sys

from utils import FindBaseNaCl, GetObjcopyCmd, get_sfi_string, shellcmd

def NewerThanOrNotThere(old_path, new_path):
    """Returns whether old_path is newer than new_path.

    Also returns true if either path doesn't exist.
    """
    if not (os.path.exists(old_path) and os.path.exists(new_path)):
        return True
    return os.path.getmtime(old_path) > os.path.getmtime(new_path)

def BuildRegex(patterns, syms):
    """Build a regular expression string for inclusion or exclusion.

    Creates a regex string from an array of patterns and an array
    of symbol names.  Each element in the patterns array is either a
    regex, or a range of entries in the symbol name array, e.g. '2:9'.
    """
    pattern_list = []
    for pattern in patterns:
        if pattern[0].isdigit() or pattern[0] == ':':
            # Legitimate symbols or regexes shouldn't start with a
            # digit or a ':', so interpret the pattern as a range.
            interval = pattern.split(':')
            if len(interval) == 1:
                # Treat singleton 'n' as 'n:n+1'.
                lower = int(interval[0])
                upper = lower + 1
            elif len(interval) == 2:
                # Handle 'a:b', 'a:', and ':b' with suitable defaults.
                lower = int(interval[0]) if len(interval[0]) else 0
                upper = int(interval[1]) if len(interval[1]) else len(syms)
            else:
                print 'Invalid range syntax: {p}'.format(p=pattern)
                exit(1)
            pattern = '$|^'.join([re.escape(p) for p in syms[lower:upper]])
        pattern_list.append('^' + pattern + '$')
    return '|'.join(pattern_list) if len(pattern_list) else '^$'

def MatchSymbol(sym, re_include, re_exclude, default_match):
    """Match a symbol name against inclusion/exclusion rules.

    Returns True or False depending on whether the given symbol
    matches the compiled include or exclude regexes.  The default is
    returned if neither the include nor the exclude regex matches.
    """
    if re_exclude.match(sym):
        # Always honor an explicit exclude before considering
        # includes.
        return False
    if re_include.match(sym):
        return True
    return default_match

def AddOptionalArgs(argparser):
    argparser.add_argument('--force', dest='force', type=int, choices=[0, 1],
                           default=1,
                           help='Force all re-translations of the pexe.' +
                                ' Default %(default)s.')
    argparser.add_argument('--include', '-i', default=[], dest='include',
                           action='append',
                           help='Subzero symbols to include ' +
                                '(regex or line range)')
    argparser.add_argument('--exclude', '-e', default=[], dest='exclude',
                           action='append',
                           help='Subzero symbols to exclude ' +
                                '(regex or line range)')
    argparser.add_argument('--output', '-o', default='a.out', dest='output',
                           action='store',
                           help='Output executable. Default %(default)s.')
    argparser.add_argument('-O', default='2', dest='optlevel',
                           choices=['m1', '-1', '0', '1', '2'],
                           help='Optimization level ' +
                                '(m1 and -1 are equivalent).' +
                                ' Default %(default)s.')
    argparser.add_argument('--filetype', default='iasm', dest='filetype',
                           choices=['obj', 'asm', 'iasm'],
                           help='Output file type.  Default %(default)s.')
    argparser.add_argument('--sandbox', dest='sandbox', action='store_true',
                           help='Enable sandboxing in the translator')
    argparser.add_argument('--nonsfi', dest='nonsfi', action='store_true',
                           help='Enable Non-SFI in the translator')
    argparser.add_argument('--enable-block-profile',
                           dest='enable_block_profile', action='store_true',
                           help='Enable basic block profiling.')
    argparser.add_argument('--target', default='x8632', dest='target',
                           choices=['arm32', 'x8632', 'x8664'],
                           help='Generate code for specified target.')
    argparser.add_argument('--verbose', '-v', dest='verbose',
                           action='store_true',
                           help='Display some extra debugging output')
    argparser.add_argument('--sz', dest='sz_args', action='append', default=[],
                           help='Extra arguments for Subzero')
    argparser.add_argument('--llc', dest='llc_args', action='append',
                           default=[], help='Extra arguments for llc')
    argparser.add_argument('--no-sz', dest='nosz', action='store_true',
                           help='Run only post-Subzero build steps')
    argparser.add_argument('--fsanitize-address', dest='asan',
                           action='store_true',
                           help='Instrument with AddressSanitizer')

def LinkSandbox(objs, exe, target, verbose=True):
    assert target in ('x8632', 'x8664', 'arm32'), \
        '-sandbox is not available for %s' % target
    nacl_root = FindBaseNaCl()
    gold = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/bin/' +
            'le32-nacl-ld.gold').format(root=nacl_root)
    target_lib_dir = {
      'arm32': 'arm',
      'x8632': 'x86-32',
      'x8664': 'x86-64',
    }[target]
    linklib = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/translator/' +
               '{target_dir}/lib').format(root=nacl_root,
                                          target_dir=target_lib_dir)
    shellcmd([gold,
              '-nostdlib',
              '--no-fix-cortex-a8',
              '--eh-frame-hdr',
              '-z', 'text',
              #'-z', 'noexecstack',
              '--build-id',
              '--entry=__pnacl_start',
              '-static', #'-pie',
              '{linklib}/crtbegin.o'.format(linklib=linklib)] +
             objs +
             [('{root}/toolchain_build/src/subzero/build/runtime/' +
               'szrt_sb_{target}.o').format(root=nacl_root, target=target),
              '{linklib}/libpnacl_irt_shim_dummy.a'.format(linklib=linklib),
              '--start-group',
              '{linklib}/libgcc.a'.format(linklib=linklib),
              '{linklib}/libcrt_platform.a'.format(linklib=linklib),
              '--end-group',
              '{linklib}/crtend.o'.format(linklib=linklib),
              '--undefined=_start',
              '--defsym=__Sz_AbsoluteZero=0',
              #'--defsym=_begin=0',
              '-o', exe
             ], echo=verbose)

def LinkNonsfi(objs, exe, target, verbose=True):
    nacl_root = FindBaseNaCl()
    gold = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/bin/' +
            'le32-nacl-ld.gold').format(root=nacl_root)
    target_lib_dir = {
      'arm32': 'arm-nonsfi',
      'x8632': 'x86-32-nonsfi',
    }[target]
    linklib = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/translator/' +
               '{target_dir}/lib').format(root=nacl_root,
                                          target_dir=target_lib_dir)
    shellcmd([gold,
              '-nostdlib',
              '--no-fix-cortex-a8',
              '--eh-frame-hdr',
              '-z', 'text',
              '-z', 'noexecstack',
              '--build-id',
              '--entry=__pnacl_start',
              '-pie',
              '{linklib}/crtbegin.o'.format(linklib=linklib)] +
             objs +
             [('{root}/toolchain_build/src/subzero/build/runtime/' +
               'szrt_nonsfi_{target}.o').format(root=nacl_root, target=target),
              '{linklib}/libpnacl_irt_shim_dummy.a'.format(linklib=linklib),
              '--start-group',
              '{linklib}/libgcc.a'.format(linklib=linklib),
              '{linklib}/libcrt_platform.a'.format(linklib=linklib),
              '--end-group',
              '{linklib}/crtend.o'.format(linklib=linklib),
              '--undefined=_start',
              '--defsym=__Sz_AbsoluteZero=0',
              '--defsym=_begin=0',
              '-o', exe
             ], echo=verbose)

def LinkNative(objs, exe, target, verbose=True):
    nacl_root = FindBaseNaCl()
    linker = {
      'arm32': '/usr/bin/arm-linux-gnueabihf-g++',
      'mips32': '/usr/bin/mipsel-linux-gnu-g++',
      'x8632': ('{root}/../third_party/llvm-build/Release+Asserts/bin/clang'
               ).format(root=nacl_root),
      'x8664': ('{root}/../third_party/llvm-build/Release+Asserts/bin/clang'
               ).format(root=nacl_root)
    }[target]

    extra_linker_args = {
      'arm32': ['-mcpu=cortex-a9'],
      'x8632': ['-m32'],
      'x8664': ['-mx32']
    }[target]

    lib_dir = {
      'arm32': 'arm-linux',
      'x8632': 'x86-32-linux',
      'x8664': 'x86-64-linux',
    }[target]

    shellcmd([linker] +
             extra_linker_args +
             objs +
             ['-o', exe,
              ('{root}/toolchain/linux_x86/pnacl_newlib_raw/translator/' +
               '{lib_dir}/lib/' +
               '{{unsandboxed_irt,irt_random,irt_query_list}}.o').format(
                   root=nacl_root, lib_dir=lib_dir),
              ('{root}/toolchain_build/src/subzero/build/runtime/' +
               'szrt_native_{target}.o').format(root=nacl_root, target=target),
              '-lm', '-lpthread', '-lrt',
              '-Wl,--defsym=__Sz_AbsoluteZero=0'
             ], echo=verbose)

def main():
    """Create a hybrid translation from Subzero and llc.

    Takes a finalized pexe and builds a native executable as a hybrid of Subzero
    and llc translated bitcode.  Linker tricks are used to determine whether
    Subzero or llc generated symbols are used, on a per-symbol basis.

    By default, for every symbol, its Subzero version is used.  Subzero and llc
    symbols can be selectively enabled/disabled via regular expressions on the
    symbol name, or by ranges of lines in this program's auto-generated symbol
    file.

    For each symbol, the --exclude arguments are first checked (the symbol is
    'rejected' on a match), followed by the --include arguments (the symbol is
    'accepted' on a match), followed by unconditional 'rejection'.  The Subzero
    version is used for an 'accepted' symbol, and the llc version is used for a
    'rejected' symbol.

    Each --include and --exclude argument can be a regular expression or a range
    of lines in the symbol file.  Each regular expression is wrapped inside
    '^$', so if you want a substring match on 'foo', use '.*foo.*' instead.
    Ranges use python-style 'first:last' notation, so e.g. use '0:10' or ':10'
    for the first 10 lines of the file, or '1' for the second line of the file.

    If no --include or --exclude arguments are given, the executable is produced
    entirely using Subzero, without using llc or linker tricks.

    When using the --force=0 option, this script uses file modification
    timestamps to determine whether llc and Subzero re-translation are needed.
    It checks timestamps of llc, pnacl-sz, and the pexe against the translated
    object files to determine the minimal work necessary.  The --force=1 option
    (default) suppresses those checks and re-translates everything.

    This script expects various PNaCl and LLVM tools to be found within the
    native_client tree.  When changes are made to these tools, copy them this
    way:
      cd native_client
      toolchain_build/toolchain_build_pnacl.py llvm_x86_64_linux \\
      --install=toolchain/linux_x86/pnacl_newlib_raw
    """
    argparser = argparse.ArgumentParser(
        description='    ' + main.__doc__,
        formatter_class=argparse.RawTextHelpFormatter)
    AddOptionalArgs(argparser)
    argparser.add_argument('pexe', help='Finalized pexe to translate')
    args = argparser.parse_args()
    pexe = args.pexe
    exe = args.output
    ProcessPexe(args, pexe, exe)

def ProcessPexe(args, pexe, exe):
    [pexe_base, ext] = os.path.splitext(pexe)
    if ext != '.pexe':
        pexe_base = pexe
    pexe_base_unescaped = pexe_base
    pexe_base = pipes.quote(pexe_base)
    pexe = pipes.quote(pexe)

    nacl_root = FindBaseNaCl()
    path_addition = (
        '{root}/toolchain/linux_x86/pnacl_newlib_raw/bin'
        ).format(root=nacl_root)
    obj_llc = pexe_base + '.llc.o'
    obj_sz = pexe_base + '.sz.o'
    asm_sz = pexe_base + '.sz.s'
    obj_llc_weak = pexe_base + '.weak.llc.o'
    obj_sz_weak = pexe_base + '.weak.sz.o'
    obj_partial = obj_sz  # overridden for hybrid mode
    sym_llc = pexe_base + '.sym.llc.txt'
    sym_sz = pexe_base + '.sym.sz.txt'
    sym_sz_unescaped = pexe_base_unescaped + '.sym.sz.txt'
    whitelist_sz = pexe_base + '.wl.sz.txt'
    whitelist_sz_unescaped = pexe_base_unescaped + '.wl.sz.txt'
    pnacl_sz = (
        '{root}/toolchain_build/src/subzero/pnacl-sz'
        ).format(root=nacl_root)
    llcbin = '{base}/pnacl-llc'.format(base=path_addition)
    gold = '{base}/le32-nacl-ld.gold'.format(base=path_addition)
    objcopy = '{base}/{objcopy}'.format(base=path_addition,
                                        objcopy=GetObjcopyCmd(args.target))
    opt_level = args.optlevel
    opt_level_map = { 'm1':'0', '-1':'0', '0':'0', '1':'1', '2':'2' }
    hybrid = args.include or args.exclude
    native = not args.sandbox and not args.nonsfi
    if args.asan:
        if args.sandbox or args.nonsfi:
            print 'Can only use AddressSanitizer with a native build'
            exit(1)
        if '-fsanitize-address' not in args.sz_args:
          args.sz_args.append('-fsanitize-address')

    if hybrid and (args.force or
                   NewerThanOrNotThere(pexe, obj_llc) or
                   NewerThanOrNotThere(llcbin, obj_llc)):
        arch = {
          'arm32': 'arm' + get_sfi_string(args, 'v7', '-nonsfi', '-nonsfi'),
          'x8632': 'x86-32' + get_sfi_string(args, '', '-nonsfi', '-linux'),
          'x8664': 'x86-64' + get_sfi_string(args, '', '', '-linux')
        }[args.target]

        # Only run pnacl-translate in hybrid mode.
        shellcmd(['{base}/pnacl-translate'.format(base=path_addition),
                  '-split-module=1',
                  '-ffunction-sections',
                  '-fdata-sections',
                  '-c',
                  '-arch',  arch,
                  '-O' + opt_level_map[opt_level],
                  '--pnacl-driver-append-LLC_FLAGS_EXTRA=-externalize',
                  '-o', obj_llc] +
                 (['--pnacl-driver-verbose'] if args.verbose else []) +
                 args.llc_args +
                 [pexe],
                 echo=args.verbose)
        if native:
            shellcmd((
                '{objcopy} --redefine-sym _start=_user_start {obj}'
                ).format(objcopy=objcopy, obj=obj_llc), echo=args.verbose)
        # Generate llc syms file for consistency, even though it's not used.
        shellcmd((
            'nm {obj} | sed -n "s/.* [a-zA-Z] //p" > {sym}'
            ).format(obj=obj_llc, sym=sym_llc), echo=args.verbose)

    if (args.force or
        NewerThanOrNotThere(pexe, obj_sz) or
        NewerThanOrNotThere(pnacl_sz, obj_sz)):
        if not args.nosz:
            # Run pnacl-sz regardless of hybrid mode.
            shellcmd([pnacl_sz,
                      '-O' + opt_level,
                      '-bitcode-format=pnacl',
                      '-filetype=' + args.filetype,
                      '-o', obj_sz if args.filetype == 'obj' else asm_sz,
                      '-target=' + args.target] +
                     (['-externalize',
                       '-ffunction-sections',
                       '-fdata-sections'] if hybrid else []) +
                     (['-sandbox'] if args.sandbox else []) +
                     (['-nonsfi'] if args.nonsfi else []) +
                     (['-enable-block-profile'] if
                          args.enable_block_profile and not args.sandbox
                          else []) +
                     args.sz_args +
                     [pexe],
                     echo=args.verbose)
        if args.filetype != 'obj':
            triple = {
              'arm32': 'arm' + get_sfi_string(args, '-nacl', '', ''),
              'x8632': 'i686' + get_sfi_string(args, '-nacl', '', ''),
              'x8664': 'x86_64' +
                        get_sfi_string(args, '-nacl', '-linux-gnux32',
                                       '-linux-gnux32'),
            }[args.target]

            shellcmd((
                '{base}/llvm-mc -triple={triple} -filetype=obj -o {obj} {asm}'
                ).format(base=path_addition, asm=asm_sz, obj=obj_sz,
                         triple=triple),
                     echo=args.verbose)
        if native:
            shellcmd((
                '{objcopy} --redefine-sym _start=_user_start {obj}'
                ).format(objcopy=objcopy, obj=obj_sz), echo=args.verbose)
        if hybrid:
            shellcmd((
                'nm {obj} | sed -n "s/.* [a-zA-Z] //p" > {sym}'
                ).format(obj=obj_sz, sym=sym_sz), echo=args.verbose)

    if hybrid:
        with open(sym_sz_unescaped) as f:
            sz_syms = f.read().splitlines()
        re_include_str = BuildRegex(args.include, sz_syms)
        re_exclude_str = BuildRegex(args.exclude, sz_syms)
        re_include = re.compile(re_include_str)
        re_exclude = re.compile(re_exclude_str)
        # If a symbol doesn't explicitly match re_include or re_exclude,
        # the default MatchSymbol() result is True, unless some --include
        # args are provided.
        default_match = not args.include

        whitelist_has_items = False
        with open(whitelist_sz_unescaped, 'w') as f:
            for sym in sz_syms:
                if MatchSymbol(sym, re_include, re_exclude, default_match):
                    f.write(sym + '\n')
                    whitelist_has_items = True
        shellcmd((
            '{objcopy} --weaken {obj} {weak}'
            ).format(objcopy=objcopy, obj=obj_sz, weak=obj_sz_weak),
            echo=args.verbose)
        if whitelist_has_items:
            # objcopy returns an error if the --weaken-symbols file is empty.
            shellcmd((
                '{objcopy} --weaken-symbols={whitelist} {obj} {weak}'
                ).format(objcopy=objcopy,
                         whitelist=whitelist_sz, obj=obj_llc,
                         weak=obj_llc_weak),
                     echo=args.verbose)
        else:
            shellcmd((
                '{objcopy} {obj} {weak}'
                ).format(objcopy=objcopy, obj=obj_llc, weak=obj_llc_weak),
                echo=args.verbose)
        obj_partial = pexe_base + '.o'
        ld = {
          'arm32': 'arm-linux-gnueabihf-ld',
          'x8632': 'ld',
          'x8664': 'ld',
        }[args.target]
        emulation = {
          'arm32': 'armelf_linux_eabi',
          'x8632': 'elf_i386',
          'x8664': 'elf32_x86_64' if not args.sandbox else 'elf_x86_64',
        }[args.target]
        shellcmd((
            '{ld} -r -m {emulation} -o {partial} {sz} {llc}'
            ).format(ld=ld, emulation=emulation, partial=obj_partial,
                     sz=obj_sz_weak, llc=obj_llc_weak),
                 echo=args.verbose)
        shellcmd((
            '{objcopy} -w --localize-symbol="*" {partial}'
            ).format(objcopy=objcopy, partial=obj_partial),
            echo=args.verbose)
        shellcmd((
            '{objcopy} --globalize-symbol={start} ' +
            '--globalize-symbol=__Sz_block_profile_info {partial}'
            ).format(objcopy=objcopy, partial=obj_partial,
                     start=get_sfi_string(args, '_start', '_start',
                                          '_user_start')),
                 echo=args.verbose)

    # Run the linker regardless of hybrid mode.
    if args.sandbox:
        LinkSandbox([obj_partial], exe, args.target, args.verbose)
    elif args.nonsfi:
        LinkNonsfi([obj_partial], exe, args.target, args.verbose)
    else:
        objs = [obj_partial]
        if args.asan:
            objs.append(
                ('{root}/toolchain_build/src/subzero/build/runtime/' +
                 'szrt_asan_{target}.o').format(root=nacl_root,
                                                target=args.target))
        LinkNative(objs, exe, args.target, args.verbose)

    # Put the extra verbose printing at the end.
    if args.verbose and hybrid:
        print 'include={regex}'.format(regex=re_include_str)
        print 'exclude={regex}'.format(regex=re_exclude_str)
        print 'default_match={dm}'.format(dm=default_match)
        print 'Number of Subzero syms = {num}'.format(num=len(sz_syms))

if __name__ == '__main__':
    main()