Home | History | Annotate | Download | only in Build
      1 import cython
      2 from Cython import __version__
      3 
      4 import re, os, sys, time
      5 try:
      6     from glob import iglob
      7 except ImportError:
      8     # Py2.4
      9     from glob import glob as iglob
     10 
     11 try:
     12     import gzip
     13     gzip_open = gzip.open
     14     gzip_ext = '.gz'
     15 except ImportError:
     16     gzip_open = open
     17     gzip_ext = ''
     18 import shutil
     19 import subprocess
     20 
     21 try:
     22     import hashlib
     23 except ImportError:
     24     import md5 as hashlib
     25 
     26 try:
     27     from io import open as io_open
     28 except ImportError:
     29     from codecs import open as io_open
     30 
     31 try:
     32     from os.path import relpath as _relpath
     33 except ImportError:
     34     # Py<2.6
     35     def _relpath(path, start=os.path.curdir):
     36         if not path:
     37             raise ValueError("no path specified")
     38         start_list = os.path.abspath(start).split(os.path.sep)
     39         path_list = os.path.abspath(path).split(os.path.sep)
     40         i = len(os.path.commonprefix([start_list, path_list]))
     41         rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
     42         if not rel_list:
     43             return os.path.curdir
     44         return os.path.join(*rel_list)
     45 
     46 
     47 from distutils.extension import Extension
     48 
     49 from Cython import Utils
     50 from Cython.Utils import cached_function, cached_method, path_exists, find_root_package_dir
     51 from Cython.Compiler.Main import Context, CompilationOptions, default_options
     52 
     53 join_path = cached_function(os.path.join)
     54 
     55 if sys.version_info[0] < 3:
     56     # stupid Py2 distutils enforces str type in list of sources
     57     _fs_encoding = sys.getfilesystemencoding()
     58     if _fs_encoding is None:
     59         _fs_encoding = sys.getdefaultencoding()
     60     def encode_filename_in_py2(filename):
     61         if isinstance(filename, unicode):
     62             return filename.encode(_fs_encoding)
     63         return filename
     64 else:
     65     def encode_filename_in_py2(filename):
     66         return filename
     67     basestring = str
     68 
     69 def extended_iglob(pattern):
     70     if '**/' in pattern:
     71         seen = set()
     72         first, rest = pattern.split('**/', 1)
     73         if first:
     74             first = iglob(first+'/')
     75         else:
     76             first = ['']
     77         for root in first:
     78             for path in extended_iglob(join_path(root, rest)):
     79                 if path not in seen:
     80                     seen.add(path)
     81                     yield path
     82             for path in extended_iglob(join_path(root, '*', '**/' + rest)):
     83                 if path not in seen:
     84                     seen.add(path)
     85                     yield path
     86     else:
     87         for path in iglob(pattern):
     88             yield path
     89 
     90 @cached_function
     91 def file_hash(filename):
     92     path = os.path.normpath(filename.encode("UTF-8"))
     93     m = hashlib.md5(str(len(path)) + ":")
     94     m.update(path)
     95     f = open(filename, 'rb')
     96     try:
     97         data = f.read(65000)
     98         while data:
     99             m.update(data)
    100             data = f.read(65000)
    101     finally:
    102         f.close()
    103     return m.hexdigest()
    104 
    105 def parse_list(s):
    106     """
    107     >>> parse_list("a b c")
    108     ['a', 'b', 'c']
    109     >>> parse_list("[a, b, c]")
    110     ['a', 'b', 'c']
    111     >>> parse_list('a " " b')
    112     ['a', ' ', 'b']
    113     >>> parse_list('[a, ",a", "a,", ",", ]')
    114     ['a', ',a', 'a,', ',']
    115     """
    116     if s[0] == '[' and s[-1] == ']':
    117         s = s[1:-1]
    118         delimiter = ','
    119     else:
    120         delimiter = ' '
    121     s, literals = strip_string_literals(s)
    122     def unquote(literal):
    123         literal = literal.strip()
    124         if literal[0] in "'\"":
    125             return literals[literal[1:-1]]
    126         else:
    127             return literal
    128     return [unquote(item) for item in s.split(delimiter) if item.strip()]
    129 
    130 transitive_str = object()
    131 transitive_list = object()
    132 
    133 distutils_settings = {
    134     'name':                 str,
    135     'sources':              list,
    136     'define_macros':        list,
    137     'undef_macros':         list,
    138     'libraries':            transitive_list,
    139     'library_dirs':         transitive_list,
    140     'runtime_library_dirs': transitive_list,
    141     'include_dirs':         transitive_list,
    142     'extra_objects':        list,
    143     'extra_compile_args':   transitive_list,
    144     'extra_link_args':      transitive_list,
    145     'export_symbols':       list,
    146     'depends':              transitive_list,
    147     'language':             transitive_str,
    148 }
    149 
    150 @cython.locals(start=long, end=long)
    151 def line_iter(source):
    152     if isinstance(source, basestring):
    153         start = 0
    154         while True:
    155             end = source.find('\n', start)
    156             if end == -1:
    157                 yield source[start:]
    158                 return
    159             yield source[start:end]
    160             start = end+1
    161     else:
    162         for line in source:
    163             yield line
    164 
    165 class DistutilsInfo(object):
    166 
    167     def __init__(self, source=None, exn=None):
    168         self.values = {}
    169         if source is not None:
    170             for line in line_iter(source):
    171                 line = line.strip()
    172                 if line != '' and line[0] != '#':
    173                     break
    174                 line = line[1:].strip()
    175                 if line[:10] == 'distutils:':
    176                     line = line[10:]
    177                     ix = line.index('=')
    178                     key = str(line[:ix].strip())
    179                     value = line[ix+1:].strip()
    180                     type = distutils_settings[key]
    181                     if type in (list, transitive_list):
    182                         value = parse_list(value)
    183                         if key == 'define_macros':
    184                             value = [tuple(macro.split('=')) for macro in value]
    185                     self.values[key] = value
    186         elif exn is not None:
    187             for key in distutils_settings:
    188                 if key in ('name', 'sources'):
    189                     continue
    190                 value = getattr(exn, key, None)
    191                 if value:
    192                     self.values[key] = value
    193 
    194     def merge(self, other):
    195         if other is None:
    196             return self
    197         for key, value in other.values.items():
    198             type = distutils_settings[key]
    199             if type is transitive_str and key not in self.values:
    200                 self.values[key] = value
    201             elif type is transitive_list:
    202                 if key in self.values:
    203                     all = self.values[key]
    204                     for v in value:
    205                         if v not in all:
    206                             all.append(v)
    207                 else:
    208                     self.values[key] = value
    209         return self
    210 
    211     def subs(self, aliases):
    212         if aliases is None:
    213             return self
    214         resolved = DistutilsInfo()
    215         for key, value in self.values.items():
    216             type = distutils_settings[key]
    217             if type in [list, transitive_list]:
    218                 new_value_list = []
    219                 for v in value:
    220                     if v in aliases:
    221                         v = aliases[v]
    222                     if isinstance(v, list):
    223                         new_value_list += v
    224                     else:
    225                         new_value_list.append(v)
    226                 value = new_value_list
    227             else:
    228                 if value in aliases:
    229                     value = aliases[value]
    230             resolved.values[key] = value
    231         return resolved
    232 
    233     def apply(self, extension):
    234         for key, value in self.values.items():
    235             type = distutils_settings[key]
    236             if type in [list, transitive_list]:
    237                 getattr(extension, key).extend(value)
    238             else:
    239                 setattr(extension, key, value)
    240 
    241 @cython.locals(start=long, q=long, single_q=long, double_q=long, hash_mark=long,
    242                end=long, k=long, counter=long, quote_len=long)
    243 def strip_string_literals(code, prefix='__Pyx_L'):
    244     """
    245     Normalizes every string literal to be of the form '__Pyx_Lxxx',
    246     returning the normalized code and a mapping of labels to
    247     string literals.
    248     """
    249     new_code = []
    250     literals = {}
    251     counter = 0
    252     start = q = 0
    253     in_quote = False
    254     hash_mark = single_q = double_q = -1
    255     code_len = len(code)
    256 
    257     while True:
    258         if hash_mark < q:
    259             hash_mark = code.find('#', q)
    260         if single_q < q:
    261             single_q = code.find("'", q)
    262         if double_q < q:
    263             double_q = code.find('"', q)
    264         q = min(single_q, double_q)
    265         if q == -1: q = max(single_q, double_q)
    266 
    267         # We're done.
    268         if q == -1 and hash_mark == -1:
    269             new_code.append(code[start:])
    270             break
    271 
    272         # Try to close the quote.
    273         elif in_quote:
    274             if code[q-1] == u'\\':
    275                 k = 2
    276                 while q >= k and code[q-k] == u'\\':
    277                     k += 1
    278                 if k % 2 == 0:
    279                     q += 1
    280                     continue
    281             if code[q] == quote_type and (quote_len == 1 or (code_len > q + 2 and quote_type == code[q+1] == code[q+2])):
    282                 counter += 1
    283                 label = "%s%s_" % (prefix, counter)
    284                 literals[label] = code[start+quote_len:q]
    285                 full_quote = code[q:q+quote_len]
    286                 new_code.append(full_quote)
    287                 new_code.append(label)
    288                 new_code.append(full_quote)
    289                 q += quote_len
    290                 in_quote = False
    291                 start = q
    292             else:
    293                 q += 1
    294 
    295         # Process comment.
    296         elif -1 != hash_mark and (hash_mark < q or q == -1):
    297             new_code.append(code[start:hash_mark+1])
    298             end = code.find('\n', hash_mark)
    299             counter += 1
    300             label = "%s%s_" % (prefix, counter)
    301             if end == -1:
    302                 end_or_none = None
    303             else:
    304                 end_or_none = end
    305             literals[label] = code[hash_mark+1:end_or_none]
    306             new_code.append(label)
    307             if end == -1:
    308                 break
    309             start = q = end
    310 
    311         # Open the quote.
    312         else:
    313             if code_len >= q+3 and (code[q] == code[q+1] == code[q+2]):
    314                 quote_len = 3
    315             else:
    316                 quote_len = 1
    317             in_quote = True
    318             quote_type = code[q]
    319             new_code.append(code[start:q])
    320             start = q
    321             q += quote_len
    322 
    323     return "".join(new_code), literals
    324 
    325 
    326 dependancy_regex = re.compile(r"(?:^from +([0-9a-zA-Z_.]+) +cimport)|"
    327                               r"(?:^cimport +([0-9a-zA-Z_.]+)\b)|"
    328                               r"(?:^cdef +extern +from +['\"]([^'\"]+)['\"])|"
    329                               r"(?:^include +['\"]([^'\"]+)['\"])", re.M)
    330 
    331 def normalize_existing(base_path, rel_paths):
    332     return normalize_existing0(os.path.dirname(base_path), tuple(set(rel_paths)))
    333 
    334 @cached_function
    335 def normalize_existing0(base_dir, rel_paths):
    336     normalized = []
    337     for rel in rel_paths:
    338         path = join_path(base_dir, rel)
    339         if path_exists(path):
    340             normalized.append(os.path.normpath(path))
    341         else:
    342             normalized.append(rel)
    343     return normalized
    344 
    345 def resolve_depends(depends, include_dirs):
    346     include_dirs = tuple(include_dirs)
    347     resolved = []
    348     for depend in depends:
    349         path = resolve_depend(depend, include_dirs)
    350         if path is not None:
    351             resolved.append(path)
    352     return resolved
    353 
    354 @cached_function
    355 def resolve_depend(depend, include_dirs):
    356     if depend[0] == '<' and depend[-1] == '>':
    357         return None
    358     for dir in include_dirs:
    359         path = join_path(dir, depend)
    360         if path_exists(path):
    361             return os.path.normpath(path)
    362     return None
    363 
    364 @cached_function
    365 def package(filename):
    366     dir = os.path.dirname(os.path.abspath(str(filename)))
    367     if dir != filename and path_exists(join_path(dir, '__init__.py')):
    368         return package(dir) + (os.path.basename(dir),)
    369     else:
    370         return ()
    371 
    372 @cached_function
    373 def fully_qualified_name(filename):
    374     module = os.path.splitext(os.path.basename(filename))[0]
    375     return '.'.join(package(filename) + (module,))
    376 
    377 
    378 @cached_function
    379 def parse_dependencies(source_filename):
    380     # Actual parsing is way to slow, so we use regular expressions.
    381     # The only catch is that we must strip comments and string
    382     # literals ahead of time.
    383     fh = Utils.open_source_file(source_filename, "rU", error_handling='ignore')
    384     try:
    385         source = fh.read()
    386     finally:
    387         fh.close()
    388     distutils_info = DistutilsInfo(source)
    389     source, literals = strip_string_literals(source)
    390     source = source.replace('\\\n', ' ').replace('\t', ' ')
    391 
    392     # TODO: pure mode
    393     cimports = []
    394     includes = []
    395     externs  = []
    396     for m in dependancy_regex.finditer(source):
    397         cimport_from, cimport, extern, include = m.groups()
    398         if cimport_from:
    399             cimports.append(cimport_from)
    400         elif cimport:
    401             cimports.append(cimport)
    402         elif extern:
    403             externs.append(literals[extern])
    404         else:
    405             includes.append(literals[include])
    406     return cimports, includes, externs, distutils_info
    407 
    408 
    409 class DependencyTree(object):
    410 
    411     def __init__(self, context, quiet=False):
    412         self.context = context
    413         self.quiet = quiet
    414         self._transitive_cache = {}
    415 
    416     def parse_dependencies(self, source_filename):
    417         return parse_dependencies(source_filename)
    418 
    419     @cached_method
    420     def included_files(self, filename):
    421         # This is messy because included files are textually included, resolving
    422         # cimports (but not includes) relative to the including file.
    423         all = set()
    424         for include in self.parse_dependencies(filename)[1]:
    425             include_path = join_path(os.path.dirname(filename), include)
    426             if not path_exists(include_path):
    427                 include_path = self.context.find_include_file(include, None)
    428             if include_path:
    429                 if '.' + os.path.sep in include_path:
    430                     include_path = os.path.normpath(include_path)
    431                 all.add(include_path)
    432                 all.update(self.included_files(include_path))
    433             elif not self.quiet:
    434                 print("Unable to locate '%s' referenced from '%s'" % (filename, include))
    435         return all
    436 
    437     @cached_method
    438     def cimports_and_externs(self, filename):
    439         # This is really ugly. Nested cimports are resolved with respect to the
    440         # includer, but includes are resolved with respect to the includee.
    441         cimports, includes, externs = self.parse_dependencies(filename)[:3]
    442         cimports = set(cimports)
    443         externs = set(externs)
    444         for include in self.included_files(filename):
    445             included_cimports, included_externs = self.cimports_and_externs(include)
    446             cimports.update(included_cimports)
    447             externs.update(included_externs)
    448         return tuple(cimports), normalize_existing(filename, externs)
    449 
    450     def cimports(self, filename):
    451         return self.cimports_and_externs(filename)[0]
    452 
    453     def package(self, filename):
    454         return package(filename)
    455 
    456     def fully_qualified_name(self, filename):
    457         return fully_qualified_name(filename)
    458 
    459     @cached_method
    460     def find_pxd(self, module, filename=None):
    461         is_relative = module[0] == '.'
    462         if is_relative and not filename:
    463             raise NotImplementedError("New relative imports.")
    464         if filename is not None:
    465             module_path = module.split('.')
    466             if is_relative:
    467                 module_path.pop(0)  # just explicitly relative
    468             package_path = list(self.package(filename))
    469             while module_path and not module_path[0]:
    470                 try:
    471                     package_path.pop()
    472                 except IndexError:
    473                     return None   # FIXME: error?
    474                 module_path.pop(0)
    475             relative = '.'.join(package_path + module_path)
    476             pxd = self.context.find_pxd_file(relative, None)
    477             if pxd:
    478                 return pxd
    479         if is_relative:
    480             return None   # FIXME: error?
    481         return self.context.find_pxd_file(module, None)
    482 
    483     @cached_method
    484     def cimported_files(self, filename):
    485         if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'):
    486             pxd_list = [filename[:-4] + '.pxd']
    487         else:
    488             pxd_list = []
    489         for module in self.cimports(filename):
    490             if module[:7] == 'cython.' or module == 'cython':
    491                 continue
    492             pxd_file = self.find_pxd(module, filename)
    493             if pxd_file is not None:
    494                 pxd_list.append(pxd_file)
    495             elif not self.quiet:
    496                 print("missing cimport in module '%s': %s" % (module, filename))
    497         return tuple(pxd_list)
    498 
    499     @cached_method
    500     def immediate_dependencies(self, filename):
    501         all = set([filename])
    502         all.update(self.cimported_files(filename))
    503         all.update(self.included_files(filename))
    504         return all
    505 
    506     def all_dependencies(self, filename):
    507         return self.transitive_merge(filename, self.immediate_dependencies, set.union)
    508 
    509     @cached_method
    510     def timestamp(self, filename):
    511         return os.path.getmtime(filename)
    512 
    513     def extract_timestamp(self, filename):
    514         return self.timestamp(filename), filename
    515 
    516     def newest_dependency(self, filename):
    517         return max([self.extract_timestamp(f) for f in self.all_dependencies(filename)])
    518 
    519     def transitive_fingerprint(self, filename, extra=None):
    520         try:
    521             m = hashlib.md5(__version__)
    522             m.update(file_hash(filename))
    523             for x in sorted(self.all_dependencies(filename)):
    524                 if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'):
    525                     m.update(file_hash(x))
    526             if extra is not None:
    527                 m.update(str(extra))
    528             return m.hexdigest()
    529         except IOError:
    530             return None
    531 
    532     def distutils_info0(self, filename):
    533         info = self.parse_dependencies(filename)[3]
    534         externs = self.cimports_and_externs(filename)[1]
    535         if externs:
    536             if 'depends' in info.values:
    537                 info.values['depends'] = list(set(info.values['depends']).union(externs))
    538             else:
    539                 info.values['depends'] = list(externs)
    540         return info
    541 
    542     def distutils_info(self, filename, aliases=None, base=None):
    543         return (self.transitive_merge(filename, self.distutils_info0, DistutilsInfo.merge)
    544             .subs(aliases)
    545             .merge(base))
    546 
    547     def transitive_merge(self, node, extract, merge):
    548         try:
    549             seen = self._transitive_cache[extract, merge]
    550         except KeyError:
    551             seen = self._transitive_cache[extract, merge] = {}
    552         return self.transitive_merge_helper(
    553             node, extract, merge, seen, {}, self.cimported_files)[0]
    554 
    555     def transitive_merge_helper(self, node, extract, merge, seen, stack, outgoing):
    556         if node in seen:
    557             return seen[node], None
    558         deps = extract(node)
    559         if node in stack:
    560             return deps, node
    561         try:
    562             stack[node] = len(stack)
    563             loop = None
    564             for next in outgoing(node):
    565                 sub_deps, sub_loop = self.transitive_merge_helper(next, extract, merge, seen, stack, outgoing)
    566                 if sub_loop is not None:
    567                     if loop is not None and stack[loop] < stack[sub_loop]:
    568                         pass
    569                     else:
    570                         loop = sub_loop
    571                 deps = merge(deps, sub_deps)
    572             if loop == node:
    573                 loop = None
    574             if loop is None:
    575                 seen[node] = deps
    576             return deps, loop
    577         finally:
    578             del stack[node]
    579 
    580 _dep_tree = None
    581 def create_dependency_tree(ctx=None, quiet=False):
    582     global _dep_tree
    583     if _dep_tree is None:
    584         if ctx is None:
    585             ctx = Context(["."], CompilationOptions(default_options))
    586         _dep_tree = DependencyTree(ctx, quiet=quiet)
    587     return _dep_tree
    588 
    589 # This may be useful for advanced users?
    590 def create_extension_list(patterns, exclude=[], ctx=None, aliases=None, quiet=False, exclude_failures=False):
    591     if not isinstance(patterns, (list, tuple)):
    592         patterns = [patterns]
    593     explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)])
    594     seen = set()
    595     deps = create_dependency_tree(ctx, quiet=quiet)
    596     to_exclude = set()
    597     if not isinstance(exclude, list):
    598         exclude = [exclude]
    599     for pattern in exclude:
    600         to_exclude.update(map(os.path.abspath, extended_iglob(pattern)))
    601     module_list = []
    602     for pattern in patterns:
    603         if isinstance(pattern, str):
    604             filepattern = pattern
    605             template = None
    606             name = '*'
    607             base = None
    608             exn_type = Extension
    609         elif isinstance(pattern, Extension):
    610             filepattern = pattern.sources[0]
    611             if os.path.splitext(filepattern)[1] not in ('.py', '.pyx'):
    612                 # ignore non-cython modules
    613                 module_list.append(pattern)
    614                 continue
    615             template = pattern
    616             name = template.name
    617             base = DistutilsInfo(exn=template)
    618             exn_type = template.__class__
    619         else:
    620             raise TypeError(pattern)
    621         for file in extended_iglob(filepattern):
    622             if os.path.abspath(file) in to_exclude:
    623                 continue
    624             pkg = deps.package(file)
    625             if '*' in name:
    626                 module_name = deps.fully_qualified_name(file)
    627                 if module_name in explicit_modules:
    628                     continue
    629             else:
    630                 module_name = name
    631             if module_name not in seen:
    632                 try:
    633                     kwds = deps.distutils_info(file, aliases, base).values
    634                 except Exception:
    635                     if exclude_failures:
    636                         continue
    637                     raise
    638                 if base is not None:
    639                     for key, value in base.values.items():
    640                         if key not in kwds:
    641                             kwds[key] = value
    642                 sources = [file]
    643                 if template is not None:
    644                     sources += template.sources[1:]
    645                 if 'sources' in kwds:
    646                     # allow users to add .c files etc.
    647                     for source in kwds['sources']:
    648                         source = encode_filename_in_py2(source)
    649                         if source not in sources:
    650                             sources.append(source)
    651                     del kwds['sources']
    652                 if 'depends' in kwds:
    653                     depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + [find_root_package_dir(file)])
    654                     if template is not None:
    655                         # Always include everything from the template.
    656                         depends = list(set(template.depends).union(set(depends)))
    657                     kwds['depends'] = depends
    658                 module_list.append(exn_type(
    659                         name=module_name,
    660                         sources=sources,
    661                         **kwds))
    662                 m = module_list[-1]
    663                 seen.add(name)
    664     return module_list
    665 
    666 # This is the user-exposed entry point.
    667 def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False,
    668               exclude_failures=False, **options):
    669     """
    670     Compile a set of source modules into C/C++ files and return a list of distutils
    671     Extension objects for them.
    672 
    673     As module list, pass either a glob pattern, a list of glob patterns or a list of
    674     Extension objects.  The latter allows you to configure the extensions separately
    675     through the normal distutils options.
    676 
    677     When using glob patterns, you can exclude certain module names explicitly
    678     by passing them into the 'exclude' option.
    679 
    680     For parallel compilation, set the 'nthreads' option to the number of
    681     concurrent builds.
    682 
    683     For a broad 'try to compile' mode that ignores compilation failures and
    684     simply excludes the failed extensions, pass 'exclude_failures=True'. Note
    685     that this only really makes sense for compiling .py files which can also
    686     be used without compilation.
    687 
    688     Additional compilation options can be passed as keyword arguments.
    689     """
    690     if 'include_path' not in options:
    691         options['include_path'] = ['.']
    692     if 'common_utility_include_dir' in options:
    693         if options.get('cache'):
    694             raise NotImplementedError("common_utility_include_dir does not yet work with caching")
    695         if not os.path.exists(options['common_utility_include_dir']):
    696             os.makedirs(options['common_utility_include_dir'])
    697     c_options = CompilationOptions(**options)
    698     cpp_options = CompilationOptions(**options); cpp_options.cplus = True
    699     ctx = c_options.create_context()
    700     options = c_options
    701     module_list = create_extension_list(
    702         module_list,
    703         exclude=exclude,
    704         ctx=ctx,
    705         quiet=quiet,
    706         exclude_failures=exclude_failures,
    707         aliases=aliases)
    708     deps = create_dependency_tree(ctx, quiet=quiet)
    709     build_dir = getattr(options, 'build_dir', None)
    710     modules_by_cfile = {}
    711     to_compile = []
    712     for m in module_list:
    713         if build_dir:
    714             root = os.path.realpath(os.path.abspath(find_root_package_dir(m.sources[0])))
    715             def copy_to_build_dir(filepath, root=root):
    716                 filepath_abs = os.path.realpath(os.path.abspath(filepath))
    717                 if os.path.isabs(filepath):
    718                     filepath = filepath_abs
    719                 if filepath_abs.startswith(root):
    720                     mod_dir = os.path.join(build_dir,
    721                             os.path.dirname(_relpath(filepath, root)))
    722                     if not os.path.isdir(mod_dir):
    723                         os.makedirs(mod_dir)
    724                     shutil.copy(filepath, mod_dir)
    725             for dep in m.depends:
    726                 copy_to_build_dir(dep)
    727 
    728         new_sources = []
    729         for source in m.sources:
    730             base, ext = os.path.splitext(source)
    731             if ext in ('.pyx', '.py'):
    732                 if m.language == 'c++':
    733                     c_file = base + '.cpp'
    734                     options = cpp_options
    735                 else:
    736                     c_file = base + '.c'
    737                     options = c_options
    738 
    739                 # setup for out of place build directory if enabled
    740                 if build_dir:
    741                     c_file = os.path.join(build_dir, c_file)
    742                     dir = os.path.dirname(c_file)
    743                     if not os.path.isdir(dir):
    744                         os.makedirs(dir)
    745 
    746                 if os.path.exists(c_file):
    747                     c_timestamp = os.path.getmtime(c_file)
    748                 else:
    749                     c_timestamp = -1
    750 
    751                 # Priority goes first to modified files, second to direct
    752                 # dependents, and finally to indirect dependents.
    753                 if c_timestamp < deps.timestamp(source):
    754                     dep_timestamp, dep = deps.timestamp(source), source
    755                     priority = 0
    756                 else:
    757                     dep_timestamp, dep = deps.newest_dependency(source)
    758                     priority = 2 - (dep in deps.immediate_dependencies(source))
    759                 if force or c_timestamp < dep_timestamp:
    760                     if not quiet:
    761                         if source == dep:
    762                             print("Compiling %s because it changed." % source)
    763                         else:
    764                             print("Compiling %s because it depends on %s." % (source, dep))
    765                     if not force and hasattr(options, 'cache'):
    766                         extra = m.language
    767                         fingerprint = deps.transitive_fingerprint(source, extra)
    768                     else:
    769                         fingerprint = None
    770                     to_compile.append((priority, source, c_file, fingerprint, quiet,
    771                                        options, not exclude_failures))
    772                 new_sources.append(c_file)
    773                 if c_file not in modules_by_cfile:
    774                     modules_by_cfile[c_file] = [m]
    775                 else:
    776                     modules_by_cfile[c_file].append(m)
    777             else:
    778                 new_sources.append(source)
    779                 if build_dir:
    780                     copy_to_build_dir(source)
    781         m.sources = new_sources
    782     if hasattr(options, 'cache'):
    783         if not os.path.exists(options.cache):
    784             os.makedirs(options.cache)
    785     to_compile.sort()
    786     if nthreads:
    787         # Requires multiprocessing (or Python >= 2.6)
    788         try:
    789             import multiprocessing
    790             pool = multiprocessing.Pool(nthreads)
    791         except (ImportError, OSError):
    792             print("multiprocessing required for parallel cythonization")
    793             nthreads = 0
    794         else:
    795             pool.map(cythonize_one_helper, to_compile)
    796     if not nthreads:
    797         for args in to_compile:
    798             cythonize_one(*args[1:])
    799     if exclude_failures:
    800         failed_modules = set()
    801         for c_file, modules in modules_by_cfile.iteritems():
    802             if not os.path.exists(c_file):
    803                 failed_modules.update(modules)
    804             elif os.path.getsize(c_file) < 200:
    805                 f = io_open(c_file, 'r', encoding='iso8859-1')
    806                 try:
    807                     if f.read(len('#error ')) == '#error ':
    808                         # dead compilation result
    809                         failed_modules.update(modules)
    810                 finally:
    811                     f.close()
    812         if failed_modules:
    813             for module in failed_modules:
    814                 module_list.remove(module)
    815             print("Failed compilations: %s" % ', '.join(sorted([
    816                 module.name for module in failed_modules])))
    817     if hasattr(options, 'cache'):
    818         cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100))
    819     # cythonize() is often followed by the (non-Python-buffered)
    820     # compiler output, flush now to avoid interleaving output.
    821     sys.stdout.flush()
    822     return module_list
    823 
    824 
    825 if os.environ.get('XML_RESULTS'):
    826     compile_result_dir = os.environ['XML_RESULTS']
    827     def record_results(func):
    828         def with_record(*args):
    829             t = time.time()
    830             success = True
    831             try:
    832                 try:
    833                     func(*args)
    834                 except:
    835                     success = False
    836             finally:
    837                 t = time.time() - t
    838                 module = fully_qualified_name(args[0])
    839                 name = "cythonize." + module
    840                 failures = 1 - success
    841                 if success:
    842                     failure_item = ""
    843                 else:
    844                     failure_item = "failure"
    845                 output = open(os.path.join(compile_result_dir, name + ".xml"), "w")
    846                 output.write("""
    847                     <?xml version="1.0" ?>
    848                     <testsuite name="%(name)s" errors="0" failures="%(failures)s" tests="1" time="%(t)s">
    849                     <testcase classname="%(name)s" name="cythonize">
    850                     %(failure_item)s
    851                     </testcase>
    852                     </testsuite>
    853                 """.strip() % locals())
    854                 output.close()
    855         return with_record
    856 else:
    857     record_results = lambda x: x
    858 
    859 # TODO: Share context? Issue: pyx processing leaks into pxd module
    860 @record_results
    861 def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True):
    862     from Cython.Compiler.Main import compile, default_options
    863     from Cython.Compiler.Errors import CompileError, PyrexError
    864 
    865     if fingerprint:
    866         if not os.path.exists(options.cache):
    867             try:
    868                 os.mkdir(options.cache)
    869             except:
    870                 if not os.path.exists(options.cache):
    871                     raise
    872         # Cython-generated c files are highly compressible.
    873         # (E.g. a compression ratio of about 10 for Sage).
    874         fingerprint_file = join_path(
    875             options.cache, "%s-%s%s" % (os.path.basename(c_file), fingerprint, gzip_ext))
    876         if os.path.exists(fingerprint_file):
    877             if not quiet:
    878                 print("Found compiled %s in cache" % pyx_file)
    879             os.utime(fingerprint_file, None)
    880             g = gzip_open(fingerprint_file, 'rb')
    881             try:
    882                 f = open(c_file, 'wb')
    883                 try:
    884                     shutil.copyfileobj(g, f)
    885                 finally:
    886                     f.close()
    887             finally:
    888                 g.close()
    889             return
    890     if not quiet:
    891         print("Cythonizing %s" % pyx_file)
    892     if options is None:
    893         options = CompilationOptions(default_options)
    894     options.output_file = c_file
    895 
    896     any_failures = 0
    897     try:
    898         result = compile([pyx_file], options)
    899         if result.num_errors > 0:
    900             any_failures = 1
    901     except (EnvironmentError, PyrexError), e:
    902         sys.stderr.write('%s\n' % e)
    903         any_failures = 1
    904         # XXX
    905         import traceback
    906         traceback.print_exc()
    907     except Exception:
    908         if raise_on_failure:
    909             raise
    910         import traceback
    911         traceback.print_exc()
    912         any_failures = 1
    913     if any_failures:
    914         if raise_on_failure:
    915             raise CompileError(None, pyx_file)
    916         elif os.path.exists(c_file):
    917             os.remove(c_file)
    918     elif fingerprint:
    919         f = open(c_file, 'rb')
    920         try:
    921             g = gzip_open(fingerprint_file, 'wb')
    922             try:
    923                 shutil.copyfileobj(f, g)
    924             finally:
    925                 g.close()
    926         finally:
    927             f.close()
    928 
    929 def cythonize_one_helper(m):
    930     import traceback
    931     try:
    932         return cythonize_one(*m[1:])
    933     except Exception:
    934         traceback.print_exc()
    935         raise
    936 
    937 def cleanup_cache(cache, target_size, ratio=.85):
    938     try:
    939         p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE)
    940         res = p.wait()
    941         if res == 0:
    942             total_size = 1024 * int(p.stdout.read().strip().split()[0])
    943             if total_size < target_size:
    944                 return
    945     except (OSError, ValueError):
    946         pass
    947     total_size = 0
    948     all = []
    949     for file in os.listdir(cache):
    950         path = join_path(cache, file)
    951         s = os.stat(path)
    952         total_size += s.st_size
    953         all.append((s.st_atime, s.st_size, path))
    954     if total_size > target_size:
    955         for time, size, file in reversed(sorted(all)):
    956             os.unlink(file)
    957             total_size -= size
    958             if total_size < target_size * ratio:
    959                 break
    960