1 # cython: language_level = 2 2 # 3 # Code output module 4 # 5 6 import cython 7 cython.declare(os=object, re=object, operator=object, 8 Naming=object, Options=object, StringEncoding=object, 9 Utils=object, SourceDescriptor=object, StringIOTree=object, 10 DebugFlags=object, basestring=object) 11 12 import os 13 import re 14 import sys 15 from string import Template 16 import operator 17 import textwrap 18 19 try: 20 import hashlib 21 except ImportError: 22 import md5 as hashlib 23 24 import Naming 25 import Options 26 import StringEncoding 27 from Cython import Utils 28 from Scanning import SourceDescriptor 29 from Cython.StringIOTree import StringIOTree 30 import DebugFlags 31 32 try: 33 from __builtin__ import basestring 34 except ImportError: 35 from builtins import str as basestring 36 37 KEYWORDS_MUST_BE_BYTES = sys.version_info < (2,7) 38 39 40 non_portable_builtins_map = { 41 # builtins that have different names in different Python versions 42 'bytes' : ('PY_MAJOR_VERSION < 3', 'str'), 43 'unicode' : ('PY_MAJOR_VERSION >= 3', 'str'), 44 'basestring' : ('PY_MAJOR_VERSION >= 3', 'str'), 45 'xrange' : ('PY_MAJOR_VERSION >= 3', 'range'), 46 'raw_input' : ('PY_MAJOR_VERSION >= 3', 'input'), 47 'BaseException' : ('PY_VERSION_HEX < 0x02050000', 'Exception'), 48 } 49 50 basicsize_builtins_map = { 51 # builtins whose type has a different tp_basicsize than sizeof(...) 52 'PyTypeObject' : 'PyHeapTypeObject', 53 } 54 55 uncachable_builtins = [ 56 # builtin names that cannot be cached because they may or may not 57 # be available at import time 58 'WindowsError', 59 ] 60 61 modifier_output_mapper = { 62 'inline': 'CYTHON_INLINE' 63 }.get 64 65 is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match 66 67 68 def get_utility_dir(): 69 # make this a function and not global variables: 70 # http://trac.cython.org/cython_trac/ticket/475 71 Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 72 return os.path.join(Cython_dir, "Utility") 73 74 75 class UtilityCodeBase(object): 76 """ 77 Support for loading utility code from a file. 78 79 Code sections in the file can be specified as follows: 80 81 ##### MyUtility.proto ##### 82 83 [proto declarations] 84 85 ##### MyUtility.init ##### 86 87 [code run at module initialization] 88 89 ##### MyUtility ##### 90 #@requires: MyOtherUtility 91 #@substitute: naming 92 93 [definitions] 94 95 for prototypes and implementation respectively. For non-python or 96 -cython files backslashes should be used instead. 5 to 30 comment 97 characters may be used on either side. 98 99 If the @cname decorator is not used and this is a CythonUtilityCode, 100 one should pass in the 'name' keyword argument to be used for name 101 mangling of such entries. 102 """ 103 104 is_cython_utility = False 105 requires = None 106 _utility_cache = {} 107 108 @classmethod 109 def _add_utility(cls, utility, type, lines, begin_lineno, tags=None): 110 if utility is None: 111 return 112 113 code = '\n'.join(lines) 114 if tags and 'substitute' in tags and tags['substitute'] == set(['naming']): 115 del tags['substitute'] 116 try: 117 code = Template(code).substitute(vars(Naming)) 118 except (KeyError, ValueError), e: 119 raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % ( 120 type, begin_lineno, e)) 121 122 # remember correct line numbers at least until after templating 123 code = '\n' * begin_lineno + code 124 125 if type == 'proto': 126 utility[0] = code 127 elif type == 'impl': 128 utility[1] = code 129 else: 130 all_tags = utility[2] 131 if KEYWORDS_MUST_BE_BYTES: 132 type = type.encode('ASCII') 133 all_tags[type] = code 134 135 if tags: 136 all_tags = utility[2] 137 for name, values in tags.items(): 138 if KEYWORDS_MUST_BE_BYTES: 139 name = name.encode('ASCII') 140 all_tags.setdefault(name, set()).update(values) 141 142 @classmethod 143 def load_utilities_from_file(cls, path): 144 utilities = cls._utility_cache.get(path) 145 if utilities: 146 return utilities 147 148 filename = os.path.join(get_utility_dir(), path) 149 _, ext = os.path.splitext(path) 150 if ext in ('.pyx', '.py', '.pxd', '.pxi'): 151 comment = '#' 152 replace_comments = re.compile(r'^\s*#.*').sub 153 else: 154 comment = '/' 155 replace_comments = re.compile(r'^\s*//.*|^\s*/\*[^*]*\*/').sub 156 match_special = re.compile( 157 (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|' 158 r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)' 159 ) % {'C':comment}).match 160 match_type = re.compile('(.+)[.](proto|impl|init|cleanup)$').match 161 162 f = Utils.open_source_file(filename, encoding='UTF-8') 163 try: 164 all_lines = f.readlines() 165 finally: 166 f.close() 167 168 utilities = {} 169 lines = [] 170 tags = {} 171 utility = type = None 172 begin_lineno = 0 173 174 for lineno, line in enumerate(all_lines): 175 m = match_special(line) 176 if m: 177 if m.group('name'): 178 cls._add_utility(utility, type, lines, begin_lineno, tags) 179 180 begin_lineno = lineno + 1 181 del lines[:] 182 tags.clear() 183 184 name = m.group('name') 185 mtype = match_type(name) 186 if mtype: 187 name, type = mtype.groups() 188 else: 189 type = 'impl' 190 utility = utilities.setdefault(name, [None, None, {}]) 191 else: 192 tags.setdefault(m.group('tag'), set()).add(m.group('value')) 193 lines.append('') # keep line number correct 194 else: 195 lines.append(replace_comments('', line).rstrip()) 196 197 if utility is None: 198 raise ValueError("Empty utility code file") 199 200 # Don't forget to add the last utility code 201 cls._add_utility(utility, type, lines, begin_lineno, tags) 202 203 cls._utility_cache[path] = utilities 204 return utilities 205 206 @classmethod 207 def load(cls, util_code_name, from_file=None, **kwargs): 208 """ 209 Load utility code from a file specified by from_file (relative to 210 Cython/Utility) and name util_code_name. If from_file is not given, 211 load it from the file util_code_name.*. There should be only one 212 file matched by this pattern. 213 """ 214 if '::' in util_code_name: 215 from_file, util_code_name = util_code_name.rsplit('::', 1) 216 if not from_file: 217 utility_dir = get_utility_dir() 218 prefix = util_code_name + '.' 219 try: 220 listing = os.listdir(utility_dir) 221 except OSError: 222 # XXX the code below assumes as 'zipimport.zipimporter' instance 223 # XXX should be easy to generalize, but too lazy right now to write it 224 import zipfile 225 global __loader__ 226 loader = __loader__ 227 archive = loader.archive 228 fileobj = zipfile.ZipFile(archive) 229 listing = [ os.path.basename(name) 230 for name in fileobj.namelist() 231 if os.path.join(archive, name).startswith(utility_dir)] 232 fileobj.close() 233 files = [ os.path.join(utility_dir, filename) 234 for filename in listing 235 if filename.startswith(prefix) ] 236 if not files: 237 raise ValueError("No match found for utility code " + util_code_name) 238 if len(files) > 1: 239 raise ValueError("More than one filename match found for utility code " + util_code_name) 240 from_file = files[0] 241 242 utilities = cls.load_utilities_from_file(from_file) 243 proto, impl, tags = utilities[util_code_name] 244 245 if tags: 246 orig_kwargs = kwargs.copy() 247 for name, values in tags.items(): 248 if name in kwargs: 249 continue 250 # only pass lists when we have to: most argument expect one value or None 251 if name == 'requires': 252 if orig_kwargs: 253 values = [cls.load(dep, from_file, **orig_kwargs) 254 for dep in sorted(values)] 255 else: 256 # dependencies are rarely unique, so use load_cached() when we can 257 values = [cls.load_cached(dep, from_file) 258 for dep in sorted(values)] 259 elif not values: 260 values = None 261 elif len(values) == 1: 262 values = values[0] 263 kwargs[name] = values 264 265 if proto is not None: 266 kwargs['proto'] = proto 267 if impl is not None: 268 kwargs['impl'] = impl 269 270 if 'name' not in kwargs: 271 kwargs['name'] = util_code_name 272 273 if 'file' not in kwargs and from_file: 274 kwargs['file'] = from_file 275 return cls(**kwargs) 276 277 @classmethod 278 def load_cached(cls, utility_code_name, from_file=None, __cache={}): 279 """ 280 Calls .load(), but using a per-type cache based on utility name and file name. 281 """ 282 key = (cls, from_file, utility_code_name) 283 try: 284 return __cache[key] 285 except KeyError: 286 pass 287 code = __cache[key] = cls.load(utility_code_name, from_file) 288 return code 289 290 @classmethod 291 def load_as_string(cls, util_code_name, from_file=None, **kwargs): 292 """ 293 Load a utility code as a string. Returns (proto, implementation) 294 """ 295 util = cls.load(util_code_name, from_file, **kwargs) 296 proto, impl = util.proto, util.impl 297 return util.format_code(proto), util.format_code(impl) 298 299 def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub): 300 """ 301 Format a code section for output. 302 """ 303 if code_string: 304 code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n' 305 return code_string 306 307 def __str__(self): 308 return "<%s(%s)" % (type(self).__name__, self.name) 309 310 def get_tree(self): 311 pass 312 313 314 class UtilityCode(UtilityCodeBase): 315 """ 316 Stores utility code to add during code generation. 317 318 See GlobalState.put_utility_code. 319 320 hashes/equals by instance 321 322 proto C prototypes 323 impl implemenation code 324 init code to call on module initialization 325 requires utility code dependencies 326 proto_block the place in the resulting file where the prototype should 327 end up 328 name name of the utility code (or None) 329 file filename of the utility code file this utility was loaded 330 from (or None) 331 """ 332 333 def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None, 334 proto_block='utility_code_proto', name=None, file=None): 335 # proto_block: Which code block to dump prototype in. See GlobalState. 336 self.proto = proto 337 self.impl = impl 338 self.init = init 339 self.cleanup = cleanup 340 self.requires = requires 341 self._cache = {} 342 self.specialize_list = [] 343 self.proto_block = proto_block 344 self.name = name 345 self.file = file 346 347 def __hash__(self): 348 return hash((self.proto, self.impl)) 349 350 def __eq__(self, other): 351 if self is other: 352 return True 353 if not isinstance(other, type(self)): 354 return False 355 356 self_proto = getattr(self, 'proto', None) 357 other_proto = getattr(other, 'proto', None) 358 return (self_proto, self.impl) == (other_proto, other.impl) 359 360 def none_or_sub(self, s, context): 361 """ 362 Format a string in this utility code with context. If None, do nothing. 363 """ 364 if s is None: 365 return None 366 return s % context 367 368 def specialize(self, pyrex_type=None, **data): 369 # Dicts aren't hashable... 370 if pyrex_type is not None: 371 data['type'] = pyrex_type.declaration_code('') 372 data['type_name'] = pyrex_type.specialization_name() 373 key = tuple(sorted(data.items())) 374 try: 375 return self._cache[key] 376 except KeyError: 377 if self.requires is None: 378 requires = None 379 else: 380 requires = [r.specialize(data) for r in self.requires] 381 382 s = self._cache[key] = UtilityCode( 383 self.none_or_sub(self.proto, data), 384 self.none_or_sub(self.impl, data), 385 self.none_or_sub(self.init, data), 386 self.none_or_sub(self.cleanup, data), 387 requires, 388 self.proto_block) 389 390 self.specialize_list.append(s) 391 return s 392 393 def inject_string_constants(self, impl, output): 394 """Replace 'PYIDENT("xyz")' by a constant Python identifier cname. 395 """ 396 replacements = {} 397 def externalise(matchobj): 398 name = matchobj.group(1) 399 try: 400 cname = replacements[name] 401 except KeyError: 402 cname = replacements[name] = output.get_interned_identifier( 403 StringEncoding.EncodedString(name)).cname 404 return cname 405 406 impl = re.sub('PYIDENT\("([^"]+)"\)', externalise, impl) 407 return bool(replacements), impl 408 409 def put_code(self, output): 410 if self.requires: 411 for dependency in self.requires: 412 output.use_utility_code(dependency) 413 if self.proto: 414 output[self.proto_block].put_or_include( 415 self.format_code(self.proto), 416 '%s_proto' % self.name) 417 if self.impl: 418 impl = self.format_code(self.impl) 419 is_specialised, impl = self.inject_string_constants(impl, output) 420 if not is_specialised: 421 # no module specific adaptations => can be reused 422 output['utility_code_def'].put_or_include( 423 impl, '%s_impl' % self.name) 424 else: 425 output['utility_code_def'].put(impl) 426 if self.init: 427 writer = output['init_globals'] 428 writer.putln("/* %s.init */" % self.name) 429 if isinstance(self.init, basestring): 430 writer.put(self.format_code(self.init)) 431 else: 432 self.init(writer, output.module_pos) 433 writer.putln(writer.error_goto_if_PyErr(output.module_pos)) 434 writer.putln() 435 if self.cleanup and Options.generate_cleanup_code: 436 writer = output['cleanup_globals'] 437 if isinstance(self.cleanup, basestring): 438 writer.put_or_include( 439 self.format_code(self.cleanup), 440 '%s_cleanup' % self.name) 441 else: 442 self.cleanup(writer, output.module_pos) 443 444 445 def sub_tempita(s, context, file=None, name=None): 446 "Run tempita on string s with given context." 447 if not s: 448 return None 449 450 if file: 451 context['__name'] = "%s:%s" % (file, name) 452 elif name: 453 context['__name'] = name 454 455 from Cython.Tempita import sub 456 return sub(s, **context) 457 458 class TempitaUtilityCode(UtilityCode): 459 def __init__(self, name=None, proto=None, impl=None, init=None, file=None, context=None, **kwargs): 460 if context is None: 461 context = {} 462 proto = sub_tempita(proto, context, file, name) 463 impl = sub_tempita(impl, context, file, name) 464 init = sub_tempita(init, context, file, name) 465 super(TempitaUtilityCode, self).__init__( 466 proto, impl, init=init, name=name, file=file, **kwargs) 467 468 def none_or_sub(self, s, context): 469 """ 470 Format a string in this utility code with context. If None, do nothing. 471 """ 472 if s is None: 473 return None 474 return sub_tempita(s, context, self.file, self.name) 475 476 477 class LazyUtilityCode(UtilityCodeBase): 478 """ 479 Utility code that calls a callback with the root code writer when 480 available. Useful when you only have 'env' but not 'code'. 481 """ 482 483 def __init__(self, callback): 484 self.callback = callback 485 486 def put_code(self, globalstate): 487 utility = self.callback(globalstate.rootwriter) 488 globalstate.use_utility_code(utility) 489 490 491 class FunctionState(object): 492 # return_label string function return point label 493 # error_label string error catch point label 494 # continue_label string loop continue point label 495 # break_label string loop break point label 496 # return_from_error_cleanup_label string 497 # label_counter integer counter for naming labels 498 # in_try_finally boolean inside try of try...finally 499 # exc_vars (string * 3) exception variables for reraise, or None 500 # can_trace boolean line tracing is supported in the current context 501 502 # Not used for now, perhaps later 503 def __init__(self, owner, names_taken=set()): 504 self.names_taken = names_taken 505 self.owner = owner 506 507 self.error_label = None 508 self.label_counter = 0 509 self.labels_used = set() 510 self.return_label = self.new_label() 511 self.new_error_label() 512 self.continue_label = None 513 self.break_label = None 514 self.yield_labels = [] 515 516 self.in_try_finally = 0 517 self.exc_vars = None 518 self.can_trace = False 519 520 self.temps_allocated = [] # of (name, type, manage_ref, static) 521 self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status 522 self.temps_used_type = {} # name -> (type, manage_ref) 523 self.temp_counter = 0 524 self.closure_temps = None 525 526 # This is used to collect temporaries, useful to find out which temps 527 # need to be privatized in parallel sections 528 self.collect_temps_stack = [] 529 530 # This is used for the error indicator, which needs to be local to the 531 # function. It used to be global, which relies on the GIL being held. 532 # However, exceptions may need to be propagated through 'nogil' 533 # sections, in which case we introduce a race condition. 534 self.should_declare_error_indicator = False 535 self.uses_error_indicator = False 536 537 # labels 538 539 def new_label(self, name=None): 540 n = self.label_counter 541 self.label_counter = n + 1 542 label = "%s%d" % (Naming.label_prefix, n) 543 if name is not None: 544 label += '_' + name 545 return label 546 547 def new_yield_label(self): 548 label = self.new_label('resume_from_yield') 549 num_and_label = (len(self.yield_labels) + 1, label) 550 self.yield_labels.append(num_and_label) 551 return num_and_label 552 553 def new_error_label(self): 554 old_err_lbl = self.error_label 555 self.error_label = self.new_label('error') 556 return old_err_lbl 557 558 def get_loop_labels(self): 559 return ( 560 self.continue_label, 561 self.break_label) 562 563 def set_loop_labels(self, labels): 564 (self.continue_label, 565 self.break_label) = labels 566 567 def new_loop_labels(self): 568 old_labels = self.get_loop_labels() 569 self.set_loop_labels( 570 (self.new_label("continue"), 571 self.new_label("break"))) 572 return old_labels 573 574 def get_all_labels(self): 575 return ( 576 self.continue_label, 577 self.break_label, 578 self.return_label, 579 self.error_label) 580 581 def set_all_labels(self, labels): 582 (self.continue_label, 583 self.break_label, 584 self.return_label, 585 self.error_label) = labels 586 587 def all_new_labels(self): 588 old_labels = self.get_all_labels() 589 new_labels = [] 590 for old_label, name in zip(old_labels, ['continue', 'break', 'return', 'error']): 591 if old_label: 592 new_labels.append(self.new_label(name)) 593 else: 594 new_labels.append(old_label) 595 self.set_all_labels(new_labels) 596 return old_labels 597 598 def use_label(self, lbl): 599 self.labels_used.add(lbl) 600 601 def label_used(self, lbl): 602 return lbl in self.labels_used 603 604 # temp handling 605 606 def allocate_temp(self, type, manage_ref, static=False): 607 """ 608 Allocates a temporary (which may create a new one or get a previously 609 allocated and released one of the same type). Type is simply registered 610 and handed back, but will usually be a PyrexType. 611 612 If type.is_pyobject, manage_ref comes into play. If manage_ref is set to 613 True, the temp will be decref-ed on return statements and in exception 614 handling clauses. Otherwise the caller has to deal with any reference 615 counting of the variable. 616 617 If not type.is_pyobject, then manage_ref will be ignored, but it 618 still has to be passed. It is recommended to pass False by convention 619 if it is known that type will never be a Python object. 620 621 static=True marks the temporary declaration with "static". 622 This is only used when allocating backing store for a module-level 623 C array literals. 624 625 A C string referring to the variable is returned. 626 """ 627 if type.is_const: 628 type = type.const_base_type 629 if not type.is_pyobject and not type.is_memoryviewslice: 630 # Make manage_ref canonical, so that manage_ref will always mean 631 # a decref is needed. 632 manage_ref = False 633 634 freelist = self.temps_free.get((type, manage_ref)) 635 if freelist is not None and len(freelist) > 0: 636 result = freelist.pop() 637 else: 638 while True: 639 self.temp_counter += 1 640 result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter) 641 if not result in self.names_taken: break 642 self.temps_allocated.append((result, type, manage_ref, static)) 643 self.temps_used_type[result] = (type, manage_ref) 644 if DebugFlags.debug_temp_code_comments: 645 self.owner.putln("/* %s allocated */" % result) 646 647 if self.collect_temps_stack: 648 self.collect_temps_stack[-1].add((result, type)) 649 650 return result 651 652 def release_temp(self, name): 653 """ 654 Releases a temporary so that it can be reused by other code needing 655 a temp of the same type. 656 """ 657 type, manage_ref = self.temps_used_type[name] 658 freelist = self.temps_free.get((type, manage_ref)) 659 if freelist is None: 660 freelist = [] 661 self.temps_free[(type, manage_ref)] = freelist 662 if name in freelist: 663 raise RuntimeError("Temp %s freed twice!" % name) 664 freelist.append(name) 665 if DebugFlags.debug_temp_code_comments: 666 self.owner.putln("/* %s released */" % name) 667 668 def temps_in_use(self): 669 """Return a list of (cname,type,manage_ref) tuples of temp names and their type 670 that are currently in use. 671 """ 672 used = [] 673 for name, type, manage_ref, static in self.temps_allocated: 674 freelist = self.temps_free.get((type, manage_ref)) 675 if freelist is None or name not in freelist: 676 used.append((name, type, manage_ref and type.is_pyobject)) 677 return used 678 679 def temps_holding_reference(self): 680 """Return a list of (cname,type) tuples of temp names and their type 681 that are currently in use. This includes only temps of a 682 Python object type which owns its reference. 683 """ 684 return [(name, type) 685 for name, type, manage_ref in self.temps_in_use() 686 if manage_ref and type.is_pyobject] 687 688 def all_managed_temps(self): 689 """Return a list of (cname, type) tuples of refcount-managed Python objects. 690 """ 691 return [(cname, type) 692 for cname, type, manage_ref, static in self.temps_allocated 693 if manage_ref] 694 695 def all_free_managed_temps(self): 696 """Return a list of (cname, type) tuples of refcount-managed Python 697 objects that are not currently in use. This is used by 698 try-except and try-finally blocks to clean up temps in the 699 error case. 700 """ 701 return [(cname, type) 702 for (type, manage_ref), freelist in self.temps_free.items() 703 if manage_ref 704 for cname in freelist] 705 706 def start_collecting_temps(self): 707 """ 708 Useful to find out which temps were used in a code block 709 """ 710 self.collect_temps_stack.append(set()) 711 712 def stop_collecting_temps(self): 713 return self.collect_temps_stack.pop() 714 715 def init_closure_temps(self, scope): 716 self.closure_temps = ClosureTempAllocator(scope) 717 718 719 class NumConst(object): 720 """Global info about a Python number constant held by GlobalState. 721 722 cname string 723 value string 724 py_type string int, long, float 725 value_code string evaluation code if different from value 726 """ 727 728 def __init__(self, cname, value, py_type, value_code=None): 729 self.cname = cname 730 self.value = value 731 self.py_type = py_type 732 self.value_code = value_code or value 733 734 735 class PyObjectConst(object): 736 """Global info about a generic constant held by GlobalState. 737 """ 738 # cname string 739 # type PyrexType 740 741 def __init__(self, cname, type): 742 self.cname = cname 743 self.type = type 744 745 746 cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object, 747 replace_identifier=object, find_alphanums=object) 748 possible_unicode_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match 749 possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match 750 replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub 751 find_alphanums = re.compile('([a-zA-Z0-9]+)').findall 752 753 class StringConst(object): 754 """Global info about a C string constant held by GlobalState. 755 """ 756 # cname string 757 # text EncodedString or BytesLiteral 758 # py_strings {(identifier, encoding) : PyStringConst} 759 760 def __init__(self, cname, text, byte_string): 761 self.cname = cname 762 self.text = text 763 self.escaped_value = StringEncoding.escape_byte_string(byte_string) 764 self.py_strings = None 765 self.py_versions = [] 766 767 def add_py_version(self, version): 768 if not version: 769 self.py_versions = [2,3] 770 elif version not in self.py_versions: 771 self.py_versions.append(version) 772 773 def get_py_string_const(self, encoding, identifier=None, 774 is_str=False, py3str_cstring=None): 775 py_strings = self.py_strings 776 text = self.text 777 778 is_str = bool(identifier or is_str) 779 is_unicode = encoding is None and not is_str 780 781 if encoding is None: 782 # unicode string 783 encoding_key = None 784 else: 785 # bytes or str 786 encoding = encoding.lower() 787 if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'): 788 encoding = None 789 encoding_key = None 790 else: 791 encoding_key = ''.join(find_alphanums(encoding)) 792 793 key = (is_str, is_unicode, encoding_key, py3str_cstring) 794 if py_strings is not None: 795 try: 796 return py_strings[key] 797 except KeyError: 798 pass 799 else: 800 self.py_strings = {} 801 802 if identifier: 803 intern = True 804 elif identifier is None: 805 if isinstance(text, unicode): 806 intern = bool(possible_unicode_identifier(text)) 807 else: 808 intern = bool(possible_bytes_identifier(text)) 809 else: 810 intern = False 811 if intern: 812 prefix = Naming.interned_prefixes['str'] 813 else: 814 prefix = Naming.py_const_prefix 815 816 if encoding_key: 817 encoding_prefix = '_%s' % encoding_key 818 else: 819 encoding_prefix = '' 820 821 pystring_cname = "%s%s%s_%s" % ( 822 prefix, 823 (is_str and 's') or (is_unicode and 'u') or 'b', 824 encoding_prefix, 825 self.cname[len(Naming.const_prefix):]) 826 827 py_string = PyStringConst( 828 pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern) 829 self.py_strings[key] = py_string 830 return py_string 831 832 class PyStringConst(object): 833 """Global info about a Python string constant held by GlobalState. 834 """ 835 # cname string 836 # py3str_cstring string 837 # encoding string 838 # intern boolean 839 # is_unicode boolean 840 # is_str boolean 841 842 def __init__(self, cname, encoding, is_unicode, is_str=False, 843 py3str_cstring=None, intern=False): 844 self.cname = cname 845 self.py3str_cstring = py3str_cstring 846 self.encoding = encoding 847 self.is_str = is_str 848 self.is_unicode = is_unicode 849 self.intern = intern 850 851 def __lt__(self, other): 852 return self.cname < other.cname 853 854 855 class GlobalState(object): 856 # filename_table {string : int} for finding filename table indexes 857 # filename_list [string] filenames in filename table order 858 # input_file_contents dict contents (=list of lines) of any file that was used as input 859 # to create this output C code. This is 860 # used to annotate the comments. 861 # 862 # utility_codes set IDs of used utility code (to avoid reinsertion) 863 # 864 # declared_cnames {string:Entry} used in a transition phase to merge pxd-declared 865 # constants etc. into the pyx-declared ones (i.e, 866 # check if constants are already added). 867 # In time, hopefully the literals etc. will be 868 # supplied directly instead. 869 # 870 # const_cnames_used dict global counter for unique constant identifiers 871 # 872 873 # parts {string:CCodeWriter} 874 875 876 # interned_strings 877 # consts 878 # interned_nums 879 880 # directives set Temporary variable used to track 881 # the current set of directives in the code generation 882 # process. 883 884 directives = {} 885 886 code_layout = [ 887 'h_code', 888 'filename_table', 889 'utility_code_proto_before_types', 890 'numeric_typedefs', # Let these detailed individual parts stay!, 891 'complex_type_declarations', # as the proper solution is to make a full DAG... 892 'type_declarations', # More coarse-grained blocks would simply hide 893 'utility_code_proto', # the ugliness, not fix it 894 'module_declarations', 895 'typeinfo', 896 'before_global_var', 897 'global_var', 898 'decls', 899 'all_the_rest', 900 'pystring_table', 901 'cached_builtins', 902 'cached_constants', 903 'init_globals', 904 'init_module', 905 'cleanup_globals', 906 'cleanup_module', 907 'main_method', 908 'utility_code_def', 909 'end' 910 ] 911 912 913 def __init__(self, writer, module_node, emit_linenums=False, common_utility_include_dir=None): 914 self.filename_table = {} 915 self.filename_list = [] 916 self.input_file_contents = {} 917 self.utility_codes = set() 918 self.declared_cnames = {} 919 self.in_utility_code_generation = False 920 self.emit_linenums = emit_linenums 921 self.common_utility_include_dir = common_utility_include_dir 922 self.parts = {} 923 self.module_node = module_node # because some utility code generation needs it 924 # (generating backwards-compatible Get/ReleaseBuffer 925 926 self.const_cnames_used = {} 927 self.string_const_index = {} 928 self.pyunicode_ptr_const_index = {} 929 self.num_const_index = {} 930 self.py_constants = [] 931 932 assert writer.globalstate is None 933 writer.globalstate = self 934 self.rootwriter = writer 935 936 def initialize_main_c_code(self): 937 rootwriter = self.rootwriter 938 for part in self.code_layout: 939 self.parts[part] = rootwriter.insertion_point() 940 941 if not Options.cache_builtins: 942 del self.parts['cached_builtins'] 943 else: 944 w = self.parts['cached_builtins'] 945 w.enter_cfunc_scope() 946 w.putln("static int __Pyx_InitCachedBuiltins(void) {") 947 948 w = self.parts['cached_constants'] 949 w.enter_cfunc_scope() 950 w.putln("") 951 w.putln("static int __Pyx_InitCachedConstants(void) {") 952 w.put_declare_refcount_context() 953 w.put_setup_refcount_context("__Pyx_InitCachedConstants") 954 955 w = self.parts['init_globals'] 956 w.enter_cfunc_scope() 957 w.putln("") 958 w.putln("static int __Pyx_InitGlobals(void) {") 959 960 if not Options.generate_cleanup_code: 961 del self.parts['cleanup_globals'] 962 else: 963 w = self.parts['cleanup_globals'] 964 w.enter_cfunc_scope() 965 w.putln("") 966 w.putln("static void __Pyx_CleanupGlobals(void) {") 967 968 # 969 # utility_code_def 970 # 971 code = self.parts['utility_code_def'] 972 if self.emit_linenums: 973 code.write('\n#line 1 "cython_utility"\n') 974 code.putln("") 975 code.putln("/* Runtime support code */") 976 977 def finalize_main_c_code(self): 978 self.close_global_decls() 979 980 # 981 # utility_code_def 982 # 983 code = self.parts['utility_code_def'] 984 code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[1]) 985 code.putln("") 986 987 def __getitem__(self, key): 988 return self.parts[key] 989 990 # 991 # Global constants, interned objects, etc. 992 # 993 def close_global_decls(self): 994 # This is called when it is known that no more global declarations will 995 # declared. 996 self.generate_const_declarations() 997 if Options.cache_builtins: 998 w = self.parts['cached_builtins'] 999 w.putln("return 0;") 1000 if w.label_used(w.error_label): 1001 w.put_label(w.error_label) 1002 w.putln("return -1;") 1003 w.putln("}") 1004 w.exit_cfunc_scope() 1005 1006 w = self.parts['cached_constants'] 1007 w.put_finish_refcount_context() 1008 w.putln("return 0;") 1009 if w.label_used(w.error_label): 1010 w.put_label(w.error_label) 1011 w.put_finish_refcount_context() 1012 w.putln("return -1;") 1013 w.putln("}") 1014 w.exit_cfunc_scope() 1015 1016 w = self.parts['init_globals'] 1017 w.putln("return 0;") 1018 if w.label_used(w.error_label): 1019 w.put_label(w.error_label) 1020 w.putln("return -1;") 1021 w.putln("}") 1022 w.exit_cfunc_scope() 1023 1024 if Options.generate_cleanup_code: 1025 w = self.parts['cleanup_globals'] 1026 w.putln("}") 1027 w.exit_cfunc_scope() 1028 1029 if Options.generate_cleanup_code: 1030 w = self.parts['cleanup_module'] 1031 w.putln("}") 1032 w.exit_cfunc_scope() 1033 1034 def put_pyobject_decl(self, entry): 1035 self['global_var'].putln("static PyObject *%s;" % entry.cname) 1036 1037 # constant handling at code generation time 1038 1039 def get_cached_constants_writer(self): 1040 return self.parts['cached_constants'] 1041 1042 def get_int_const(self, str_value, longness=False): 1043 py_type = longness and 'long' or 'int' 1044 try: 1045 c = self.num_const_index[(str_value, py_type)] 1046 except KeyError: 1047 c = self.new_num_const(str_value, py_type) 1048 return c 1049 1050 def get_float_const(self, str_value, value_code): 1051 try: 1052 c = self.num_const_index[(str_value, 'float')] 1053 except KeyError: 1054 c = self.new_num_const(str_value, 'float', value_code) 1055 return c 1056 1057 def get_py_const(self, type, prefix='', cleanup_level=None): 1058 # create a new Python object constant 1059 const = self.new_py_const(type, prefix) 1060 if cleanup_level is not None \ 1061 and cleanup_level <= Options.generate_cleanup_code: 1062 cleanup_writer = self.parts['cleanup_globals'] 1063 cleanup_writer.putln('Py_CLEAR(%s);' % const.cname) 1064 return const 1065 1066 def get_string_const(self, text, py_version=None): 1067 # return a C string constant, creating a new one if necessary 1068 if text.is_unicode: 1069 byte_string = text.utf8encode() 1070 else: 1071 byte_string = text.byteencode() 1072 try: 1073 c = self.string_const_index[byte_string] 1074 except KeyError: 1075 c = self.new_string_const(text, byte_string) 1076 c.add_py_version(py_version) 1077 return c 1078 1079 def get_pyunicode_ptr_const(self, text): 1080 # return a Py_UNICODE[] constant, creating a new one if necessary 1081 assert text.is_unicode 1082 try: 1083 c = self.pyunicode_ptr_const_index[text] 1084 except KeyError: 1085 c = self.pyunicode_ptr_const_index[text] = self.new_const_cname() 1086 return c 1087 1088 def get_py_string_const(self, text, identifier=None, 1089 is_str=False, unicode_value=None): 1090 # return a Python string constant, creating a new one if necessary 1091 py3str_cstring = None 1092 if is_str and unicode_value is not None \ 1093 and unicode_value.utf8encode() != text.byteencode(): 1094 py3str_cstring = self.get_string_const(unicode_value, py_version=3) 1095 c_string = self.get_string_const(text, py_version=2) 1096 else: 1097 c_string = self.get_string_const(text) 1098 py_string = c_string.get_py_string_const( 1099 text.encoding, identifier, is_str, py3str_cstring) 1100 return py_string 1101 1102 def get_interned_identifier(self, text): 1103 return self.get_py_string_const(text, identifier=True) 1104 1105 def new_string_const(self, text, byte_string): 1106 cname = self.new_string_const_cname(byte_string) 1107 c = StringConst(cname, text, byte_string) 1108 self.string_const_index[byte_string] = c 1109 return c 1110 1111 def new_num_const(self, value, py_type, value_code=None): 1112 cname = self.new_num_const_cname(value, py_type) 1113 c = NumConst(cname, value, py_type, value_code) 1114 self.num_const_index[(value, py_type)] = c 1115 return c 1116 1117 def new_py_const(self, type, prefix=''): 1118 cname = self.new_const_cname(prefix) 1119 c = PyObjectConst(cname, type) 1120 self.py_constants.append(c) 1121 return c 1122 1123 def new_string_const_cname(self, bytes_value): 1124 # Create a new globally-unique nice name for a C string constant. 1125 value = bytes_value.decode('ASCII', 'ignore') 1126 return self.new_const_cname(value=value) 1127 1128 def new_num_const_cname(self, value, py_type): 1129 if py_type == 'long': 1130 value += 'L' 1131 py_type = 'int' 1132 prefix = Naming.interned_prefixes[py_type] 1133 cname = "%s%s" % (prefix, value) 1134 cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_') 1135 return cname 1136 1137 def new_const_cname(self, prefix='', value=''): 1138 value = replace_identifier('_', value)[:32].strip('_') 1139 used = self.const_cnames_used 1140 name_suffix = value 1141 while name_suffix in used: 1142 counter = used[value] = used[value] + 1 1143 name_suffix = '%s_%d' % (value, counter) 1144 used[name_suffix] = 1 1145 if prefix: 1146 prefix = Naming.interned_prefixes[prefix] 1147 else: 1148 prefix = Naming.const_prefix 1149 return "%s%s" % (prefix, name_suffix) 1150 1151 def add_cached_builtin_decl(self, entry): 1152 if entry.is_builtin and entry.is_const: 1153 if self.should_declare(entry.cname, entry): 1154 self.put_pyobject_decl(entry) 1155 w = self.parts['cached_builtins'] 1156 condition = None 1157 if entry.name in non_portable_builtins_map: 1158 condition, replacement = non_portable_builtins_map[entry.name] 1159 w.putln('#if %s' % condition) 1160 self.put_cached_builtin_init( 1161 entry.pos, StringEncoding.EncodedString(replacement), 1162 entry.cname) 1163 w.putln('#else') 1164 self.put_cached_builtin_init( 1165 entry.pos, StringEncoding.EncodedString(entry.name), 1166 entry.cname) 1167 if condition: 1168 w.putln('#endif') 1169 1170 def put_cached_builtin_init(self, pos, name, cname): 1171 w = self.parts['cached_builtins'] 1172 interned_cname = self.get_interned_identifier(name).cname 1173 self.use_utility_code( 1174 UtilityCode.load_cached("GetBuiltinName", "ObjectHandling.c")) 1175 w.putln('%s = __Pyx_GetBuiltinName(%s); if (!%s) %s' % ( 1176 cname, 1177 interned_cname, 1178 cname, 1179 w.error_goto(pos))) 1180 1181 def generate_const_declarations(self): 1182 self.generate_string_constants() 1183 self.generate_num_constants() 1184 self.generate_object_constant_decls() 1185 1186 def generate_object_constant_decls(self): 1187 consts = [ (len(c.cname), c.cname, c) 1188 for c in self.py_constants ] 1189 consts.sort() 1190 decls_writer = self.parts['decls'] 1191 for _, cname, c in consts: 1192 decls_writer.putln( 1193 "static %s;" % c.type.declaration_code(cname)) 1194 1195 def generate_string_constants(self): 1196 c_consts = [ (len(c.cname), c.cname, c) 1197 for c in self.string_const_index.values() ] 1198 c_consts.sort() 1199 py_strings = [] 1200 1201 decls_writer = self.parts['decls'] 1202 for _, cname, c in c_consts: 1203 conditional = False 1204 if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versions): 1205 conditional = True 1206 decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % ( 1207 (2 in c.py_versions) and '<' or '>=')) 1208 decls_writer.putln('static char %s[] = "%s";' % ( 1209 cname, StringEncoding.split_string_literal(c.escaped_value))) 1210 if conditional: 1211 decls_writer.putln("#endif") 1212 if c.py_strings is not None: 1213 for py_string in c.py_strings.values(): 1214 py_strings.append((c.cname, len(py_string.cname), py_string)) 1215 1216 for c, cname in self.pyunicode_ptr_const_index.items(): 1217 utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c) 1218 if utf16_array: 1219 # Narrow and wide representations differ 1220 decls_writer.putln("#ifdef Py_UNICODE_WIDE") 1221 decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf32_array)) 1222 if utf16_array: 1223 decls_writer.putln("#else") 1224 decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array)) 1225 decls_writer.putln("#endif") 1226 1227 if py_strings: 1228 self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c")) 1229 py_strings.sort() 1230 w = self.parts['pystring_table'] 1231 w.putln("") 1232 w.putln("static __Pyx_StringTabEntry %s[] = {" % 1233 Naming.stringtab_cname) 1234 for c_cname, _, py_string in py_strings: 1235 if not py_string.is_str or not py_string.encoding or \ 1236 py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII', 1237 'UTF8', 'UTF-8'): 1238 encoding = '0' 1239 else: 1240 encoding = '"%s"' % py_string.encoding.lower() 1241 1242 decls_writer.putln( 1243 "static PyObject *%s;" % py_string.cname) 1244 if py_string.py3str_cstring: 1245 w.putln("#if PY_MAJOR_VERSION >= 3") 1246 w.putln( 1247 "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( 1248 py_string.cname, 1249 py_string.py3str_cstring.cname, 1250 py_string.py3str_cstring.cname, 1251 '0', 1, 0, 1252 py_string.intern 1253 )) 1254 w.putln("#else") 1255 w.putln( 1256 "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( 1257 py_string.cname, 1258 c_cname, 1259 c_cname, 1260 encoding, 1261 py_string.is_unicode, 1262 py_string.is_str, 1263 py_string.intern 1264 )) 1265 if py_string.py3str_cstring: 1266 w.putln("#endif") 1267 w.putln("{0, 0, 0, 0, 0, 0, 0}") 1268 w.putln("};") 1269 1270 init_globals = self.parts['init_globals'] 1271 init_globals.putln( 1272 "if (__Pyx_InitStrings(%s) < 0) %s;" % ( 1273 Naming.stringtab_cname, 1274 init_globals.error_goto(self.module_pos))) 1275 1276 def generate_num_constants(self): 1277 consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c) 1278 for c in self.num_const_index.values()] 1279 consts.sort() 1280 decls_writer = self.parts['decls'] 1281 init_globals = self.parts['init_globals'] 1282 for py_type, _, _, value, value_code, c in consts: 1283 cname = c.cname 1284 decls_writer.putln("static PyObject *%s;" % cname) 1285 if py_type == 'float': 1286 function = 'PyFloat_FromDouble(%s)' 1287 elif py_type == 'long': 1288 function = 'PyLong_FromString((char *)"%s", 0, 0)' 1289 elif Utils.long_literal(value): 1290 function = 'PyInt_FromString((char *)"%s", 0, 0)' 1291 elif len(value.lstrip('-')) > 4: 1292 function = "PyInt_FromLong(%sL)" 1293 else: 1294 function = "PyInt_FromLong(%s)" 1295 init_globals.putln('%s = %s; %s' % ( 1296 cname, function % value_code, 1297 init_globals.error_goto_if_null(cname, self.module_pos))) 1298 1299 # The functions below are there in a transition phase only 1300 # and will be deprecated. They are called from Nodes.BlockNode. 1301 # The copy&paste duplication is intentional in order to be able 1302 # to see quickly how BlockNode worked, until this is replaced. 1303 1304 def should_declare(self, cname, entry): 1305 if cname in self.declared_cnames: 1306 other = self.declared_cnames[cname] 1307 assert str(entry.type) == str(other.type) 1308 assert entry.init == other.init 1309 return False 1310 else: 1311 self.declared_cnames[cname] = entry 1312 return True 1313 1314 # 1315 # File name state 1316 # 1317 1318 def lookup_filename(self, filename): 1319 try: 1320 index = self.filename_table[filename] 1321 except KeyError: 1322 index = len(self.filename_list) 1323 self.filename_list.append(filename) 1324 self.filename_table[filename] = index 1325 return index 1326 1327 def commented_file_contents(self, source_desc): 1328 try: 1329 return self.input_file_contents[source_desc] 1330 except KeyError: 1331 pass 1332 source_file = source_desc.get_lines(encoding='ASCII', 1333 error_handling='ignore') 1334 try: 1335 F = [u' * ' + line.rstrip().replace( 1336 u'*/', u'*[inserted by cython to avoid comment closer]/' 1337 ).replace( 1338 u'/*', u'/[inserted by cython to avoid comment start]*' 1339 ) 1340 for line in source_file] 1341 finally: 1342 if hasattr(source_file, 'close'): 1343 source_file.close() 1344 if not F: F.append(u'') 1345 self.input_file_contents[source_desc] = F 1346 return F 1347 1348 # 1349 # Utility code state 1350 # 1351 1352 def use_utility_code(self, utility_code): 1353 """ 1354 Adds code to the C file. utility_code should 1355 a) implement __eq__/__hash__ for the purpose of knowing whether the same 1356 code has already been included 1357 b) implement put_code, which takes a globalstate instance 1358 1359 See UtilityCode. 1360 """ 1361 if utility_code not in self.utility_codes: 1362 self.utility_codes.add(utility_code) 1363 utility_code.put_code(self) 1364 1365 1366 def funccontext_property(name): 1367 attribute_of = operator.attrgetter(name) 1368 def get(self): 1369 return attribute_of(self.funcstate) 1370 def set(self, value): 1371 setattr(self.funcstate, name, value) 1372 return property(get, set) 1373 1374 1375 class CCodeWriter(object): 1376 """ 1377 Utility class to output C code. 1378 1379 When creating an insertion point one must care about the state that is 1380 kept: 1381 - formatting state (level, bol) is cloned and used in insertion points 1382 as well 1383 - labels, temps, exc_vars: One must construct a scope in which these can 1384 exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for 1385 sanity checking and forward compatabilty). Created insertion points 1386 looses this scope and cannot access it. 1387 - marker: Not copied to insertion point 1388 - filename_table, filename_list, input_file_contents: All codewriters 1389 coming from the same root share the same instances simultaneously. 1390 """ 1391 1392 # f file output file 1393 # buffer StringIOTree 1394 1395 # level int indentation level 1396 # bol bool beginning of line? 1397 # marker string comment to emit before next line 1398 # funcstate FunctionState contains state local to a C function used for code 1399 # generation (labels and temps state etc.) 1400 # globalstate GlobalState contains state global for a C file (input file info, 1401 # utility code, declared constants etc.) 1402 # emit_linenums boolean whether or not to write #line pragmas 1403 # 1404 # c_line_in_traceback boolean append the c file and line number to the traceback for exceptions 1405 # 1406 # pyclass_stack list used during recursive code generation to pass information 1407 # about the current class one is in 1408 1409 globalstate = None 1410 1411 def __init__(self, create_from=None, buffer=None, copy_formatting=False, emit_linenums=None, c_line_in_traceback=True): 1412 if buffer is None: buffer = StringIOTree() 1413 self.buffer = buffer 1414 self.marker = None 1415 self.last_marker_line = 0 1416 self.source_desc = "" 1417 self.pyclass_stack = [] 1418 1419 self.funcstate = None 1420 self.level = 0 1421 self.call_level = 0 1422 self.bol = 1 1423 1424 if create_from is not None: 1425 # Use same global state 1426 self.globalstate = create_from.globalstate 1427 self.funcstate = create_from.funcstate 1428 # Clone formatting state 1429 if copy_formatting: 1430 self.level = create_from.level 1431 self.bol = create_from.bol 1432 self.call_level = create_from.call_level 1433 1434 if emit_linenums is None and self.globalstate: 1435 self.emit_linenums = self.globalstate.emit_linenums 1436 else: 1437 self.emit_linenums = emit_linenums 1438 self.c_line_in_traceback = c_line_in_traceback 1439 1440 def create_new(self, create_from, buffer, copy_formatting): 1441 # polymorphic constructor -- very slightly more versatile 1442 # than using __class__ 1443 result = CCodeWriter(create_from, buffer, copy_formatting, 1444 c_line_in_traceback=self.c_line_in_traceback) 1445 return result 1446 1447 def copyto(self, f): 1448 self.buffer.copyto(f) 1449 1450 def getvalue(self): 1451 return self.buffer.getvalue() 1452 1453 def write(self, s): 1454 # also put invalid markers (lineno 0), to indicate that those lines 1455 # have no Cython source code correspondence 1456 if self.marker is None: 1457 cython_lineno = self.last_marker_line 1458 else: 1459 cython_lineno = self.marker[0] 1460 1461 self.buffer.markers.extend([cython_lineno] * s.count('\n')) 1462 self.buffer.write(s) 1463 1464 def insertion_point(self): 1465 other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True) 1466 return other 1467 1468 def new_writer(self): 1469 """ 1470 Creates a new CCodeWriter connected to the same global state, which 1471 can later be inserted using insert. 1472 """ 1473 return CCodeWriter(create_from=self, c_line_in_traceback=self.c_line_in_traceback) 1474 1475 def insert(self, writer): 1476 """ 1477 Inserts the contents of another code writer (created with 1478 the same global state) in the current location. 1479 1480 It is ok to write to the inserted writer also after insertion. 1481 """ 1482 assert writer.globalstate is self.globalstate 1483 self.buffer.insert(writer.buffer) 1484 1485 # Properties delegated to function scope 1486 label_counter = funccontext_property("label_counter") 1487 return_label = funccontext_property("return_label") 1488 error_label = funccontext_property("error_label") 1489 labels_used = funccontext_property("labels_used") 1490 continue_label = funccontext_property("continue_label") 1491 break_label = funccontext_property("break_label") 1492 return_from_error_cleanup_label = funccontext_property("return_from_error_cleanup_label") 1493 yield_labels = funccontext_property("yield_labels") 1494 1495 # Functions delegated to function scope 1496 def new_label(self, name=None): return self.funcstate.new_label(name) 1497 def new_error_label(self): return self.funcstate.new_error_label() 1498 def new_yield_label(self): return self.funcstate.new_yield_label() 1499 def get_loop_labels(self): return self.funcstate.get_loop_labels() 1500 def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels) 1501 def new_loop_labels(self): return self.funcstate.new_loop_labels() 1502 def get_all_labels(self): return self.funcstate.get_all_labels() 1503 def set_all_labels(self, labels): return self.funcstate.set_all_labels(labels) 1504 def all_new_labels(self): return self.funcstate.all_new_labels() 1505 def use_label(self, lbl): return self.funcstate.use_label(lbl) 1506 def label_used(self, lbl): return self.funcstate.label_used(lbl) 1507 1508 1509 def enter_cfunc_scope(self): 1510 self.funcstate = FunctionState(self) 1511 1512 def exit_cfunc_scope(self): 1513 self.funcstate = None 1514 1515 # constant handling 1516 1517 def get_py_int(self, str_value, longness): 1518 return self.globalstate.get_int_const(str_value, longness).cname 1519 1520 def get_py_float(self, str_value, value_code): 1521 return self.globalstate.get_float_const(str_value, value_code).cname 1522 1523 def get_py_const(self, type, prefix='', cleanup_level=None): 1524 return self.globalstate.get_py_const(type, prefix, cleanup_level).cname 1525 1526 def get_string_const(self, text): 1527 return self.globalstate.get_string_const(text).cname 1528 1529 def get_pyunicode_ptr_const(self, text): 1530 return self.globalstate.get_pyunicode_ptr_const(text) 1531 1532 def get_py_string_const(self, text, identifier=None, 1533 is_str=False, unicode_value=None): 1534 return self.globalstate.get_py_string_const( 1535 text, identifier, is_str, unicode_value).cname 1536 1537 def get_argument_default_const(self, type): 1538 return self.globalstate.get_py_const(type).cname 1539 1540 def intern(self, text): 1541 return self.get_py_string_const(text) 1542 1543 def intern_identifier(self, text): 1544 return self.get_py_string_const(text, identifier=True) 1545 1546 def get_cached_constants_writer(self): 1547 return self.globalstate.get_cached_constants_writer() 1548 1549 # code generation 1550 1551 def putln(self, code="", safe=False): 1552 if self.marker and self.bol: 1553 self.emit_marker() 1554 if self.emit_linenums and self.last_marker_line != 0: 1555 self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc)) 1556 1557 if code: 1558 if safe: 1559 self.put_safe(code) 1560 else: 1561 self.put(code) 1562 self.write("\n") 1563 self.bol = 1 1564 1565 def emit_marker(self): 1566 self.write("\n") 1567 self.indent() 1568 self.write("/* %s */\n" % self.marker[1]) 1569 if (self.funcstate and self.funcstate.can_trace 1570 and self.globalstate.directives['linetrace']): 1571 self.indent() 1572 self.write('__Pyx_TraceLine(%d)\n' % self.marker[0]) 1573 self.last_marker_line = self.marker[0] 1574 self.marker = None 1575 1576 def put_safe(self, code): 1577 # put code, but ignore {} 1578 self.write(code) 1579 self.bol = 0 1580 1581 def put_or_include(self, code, name): 1582 include_dir = self.globalstate.common_utility_include_dir 1583 if include_dir and len(code) > 1024: 1584 include_file = "%s_%s.h" % ( 1585 name, hashlib.md5(code.encode('utf8')).hexdigest()) 1586 path = os.path.join(include_dir, include_file) 1587 if not os.path.exists(path): 1588 tmp_path = '%s.tmp%s' % (path, os.getpid()) 1589 f = Utils.open_new_file(tmp_path) 1590 try: 1591 f.write(code) 1592 finally: 1593 f.close() 1594 os.rename(tmp_path, path) 1595 code = '#include "%s"\n' % path 1596 self.put(code) 1597 1598 def put(self, code): 1599 if is_self_assignment(code): 1600 return 1601 fix_indent = False 1602 if "{" in code: 1603 dl = code.count("{") 1604 else: 1605 dl = 0 1606 if "}" in code: 1607 dl -= code.count("}") 1608 if dl < 0: 1609 self.level += dl 1610 elif dl == 0 and code[0] == "}": 1611 # special cases like "} else {" need a temporary dedent 1612 fix_indent = True 1613 self.level -= 1 1614 if self.bol: 1615 self.indent() 1616 self.write(code) 1617 self.bol = 0 1618 if dl > 0: 1619 self.level += dl 1620 elif fix_indent: 1621 self.level += 1 1622 1623 def putln_tempita(self, code, **context): 1624 from Cython.Tempita import sub 1625 self.putln(sub(code, **context)) 1626 1627 def put_tempita(self, code, **context): 1628 from Cython.Tempita import sub 1629 self.put(sub(code, **context)) 1630 1631 def increase_indent(self): 1632 self.level += 1 1633 1634 def decrease_indent(self): 1635 self.level -= 1 1636 1637 def begin_block(self): 1638 self.putln("{") 1639 self.increase_indent() 1640 1641 def end_block(self): 1642 self.decrease_indent() 1643 self.putln("}") 1644 1645 def indent(self): 1646 self.write(" " * self.level) 1647 1648 def get_py_version_hex(self, pyversion): 1649 return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4] 1650 1651 def mark_pos(self, pos): 1652 if pos is None: 1653 return 1654 source_desc, line, col = pos 1655 if self.last_marker_line == line: 1656 return 1657 assert isinstance(source_desc, SourceDescriptor) 1658 contents = self.globalstate.commented_file_contents(source_desc) 1659 lines = contents[max(0, line-3):line] # line numbers start at 1 1660 lines[-1] += u' # <<<<<<<<<<<<<<' 1661 lines += contents[line:line+2] 1662 1663 marker = u'"%s":%d\n%s\n' % ( 1664 source_desc.get_escaped_description(), line, u'\n'.join(lines)) 1665 self.marker = (line, marker) 1666 if self.emit_linenums: 1667 self.source_desc = source_desc.get_escaped_description() 1668 1669 def put_label(self, lbl): 1670 if lbl in self.funcstate.labels_used: 1671 self.putln("%s:;" % lbl) 1672 1673 def put_goto(self, lbl): 1674 self.funcstate.use_label(lbl) 1675 self.putln("goto %s;" % lbl) 1676 1677 def put_var_declaration(self, entry, storage_class="", 1678 dll_linkage=None, definition=True): 1679 #print "Code.put_var_declaration:", entry.name, "definition =", definition ### 1680 if entry.visibility == 'private' and not (definition or entry.defined_in_pxd): 1681 #print "...private and not definition, skipping", entry.cname ### 1682 return 1683 if entry.visibility == "private" and not entry.used: 1684 #print "...private and not used, skipping", entry.cname ### 1685 return 1686 if storage_class: 1687 self.put("%s " % storage_class) 1688 if not entry.cf_used: 1689 self.put('CYTHON_UNUSED ') 1690 self.put(entry.type.declaration_code( 1691 entry.cname, dll_linkage=dll_linkage)) 1692 if entry.init is not None: 1693 self.put_safe(" = %s" % entry.type.literal_code(entry.init)) 1694 elif entry.type.is_pyobject: 1695 self.put(" = NULL") 1696 self.putln(";") 1697 1698 def put_temp_declarations(self, func_context): 1699 for name, type, manage_ref, static in func_context.temps_allocated: 1700 decl = type.declaration_code(name) 1701 if type.is_pyobject: 1702 self.putln("%s = NULL;" % decl) 1703 elif type.is_memoryviewslice: 1704 import MemoryView 1705 self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init)) 1706 else: 1707 self.putln("%s%s;" % (static and "static " or "", decl)) 1708 1709 if func_context.should_declare_error_indicator: 1710 if self.funcstate.uses_error_indicator: 1711 unused = '' 1712 else: 1713 unused = 'CYTHON_UNUSED ' 1714 # Initialize these variables to silence compiler warnings 1715 self.putln("%sint %s = 0;" % (unused, Naming.lineno_cname)) 1716 self.putln("%sconst char *%s = NULL;" % (unused, Naming.filename_cname)) 1717 self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname)) 1718 1719 def put_h_guard(self, guard): 1720 self.putln("#ifndef %s" % guard) 1721 self.putln("#define %s" % guard) 1722 1723 def unlikely(self, cond): 1724 if Options.gcc_branch_hints: 1725 return 'unlikely(%s)' % cond 1726 else: 1727 return cond 1728 1729 def build_function_modifiers(self, modifiers, mapper=modifier_output_mapper): 1730 if not modifiers: 1731 return '' 1732 return '%s ' % ' '.join([mapper(m,m) for m in modifiers]) 1733 1734 # Python objects and reference counting 1735 1736 def entry_as_pyobject(self, entry): 1737 type = entry.type 1738 if (not entry.is_self_arg and not entry.type.is_complete() 1739 or entry.type.is_extension_type): 1740 return "(PyObject *)" + entry.cname 1741 else: 1742 return entry.cname 1743 1744 def as_pyobject(self, cname, type): 1745 from PyrexTypes import py_object_type, typecast 1746 return typecast(py_object_type, type, cname) 1747 1748 def put_gotref(self, cname): 1749 self.putln("__Pyx_GOTREF(%s);" % cname) 1750 1751 def put_giveref(self, cname): 1752 self.putln("__Pyx_GIVEREF(%s);" % cname) 1753 1754 def put_xgiveref(self, cname): 1755 self.putln("__Pyx_XGIVEREF(%s);" % cname) 1756 1757 def put_xgotref(self, cname): 1758 self.putln("__Pyx_XGOTREF(%s);" % cname) 1759 1760 def put_incref(self, cname, type, nanny=True): 1761 if nanny: 1762 self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type)) 1763 else: 1764 self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) 1765 1766 def put_decref(self, cname, type, nanny=True): 1767 self._put_decref(cname, type, nanny, null_check=False, clear=False) 1768 1769 def put_var_gotref(self, entry): 1770 if entry.type.is_pyobject: 1771 self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry)) 1772 1773 def put_var_giveref(self, entry): 1774 if entry.type.is_pyobject: 1775 self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry)) 1776 1777 def put_var_xgotref(self, entry): 1778 if entry.type.is_pyobject: 1779 self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry)) 1780 1781 def put_var_xgiveref(self, entry): 1782 if entry.type.is_pyobject: 1783 self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry)) 1784 1785 def put_var_incref(self, entry): 1786 if entry.type.is_pyobject: 1787 self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry)) 1788 1789 def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False): 1790 self._put_decref(cname, type, nanny, null_check=False, 1791 clear=True, clear_before_decref=clear_before_decref) 1792 1793 def put_xdecref(self, cname, type, nanny=True, have_gil=True): 1794 self._put_decref(cname, type, nanny, null_check=True, 1795 have_gil=have_gil, clear=False) 1796 1797 def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False): 1798 self._put_decref(cname, type, nanny, null_check=True, 1799 clear=True, clear_before_decref=clear_before_decref) 1800 1801 def _put_decref(self, cname, type, nanny=True, null_check=False, 1802 have_gil=True, clear=False, clear_before_decref=False): 1803 if type.is_memoryviewslice: 1804 self.put_xdecref_memoryviewslice(cname, have_gil=have_gil) 1805 return 1806 1807 prefix = nanny and '__Pyx' or 'Py' 1808 X = null_check and 'X' or '' 1809 1810 if clear: 1811 if clear_before_decref: 1812 if not nanny: 1813 X = '' # CPython doesn't have a Py_XCLEAR() 1814 self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) 1815 else: 1816 self.putln("%s_%sDECREF(%s); %s = 0;" % ( 1817 prefix, X, self.as_pyobject(cname, type), cname)) 1818 else: 1819 self.putln("%s_%sDECREF(%s);" % ( 1820 prefix, X, self.as_pyobject(cname, type))) 1821 1822 def put_decref_set(self, cname, rhs_cname): 1823 self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) 1824 1825 def put_xdecref_set(self, cname, rhs_cname): 1826 self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) 1827 1828 def put_var_decref(self, entry): 1829 if entry.type.is_pyobject: 1830 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) 1831 1832 def put_var_xdecref(self, entry): 1833 if entry.type.is_pyobject: 1834 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) 1835 1836 def put_var_decref_clear(self, entry): 1837 self._put_var_decref_clear(entry, null_check=False) 1838 1839 def put_var_xdecref_clear(self, entry): 1840 self._put_var_decref_clear(entry, null_check=True) 1841 1842 def _put_var_decref_clear(self, entry, null_check): 1843 if entry.type.is_pyobject: 1844 if entry.in_closure: 1845 # reset before DECREF to make sure closure state is 1846 # consistent during call to DECREF() 1847 self.putln("__Pyx_%sCLEAR(%s);" % ( 1848 null_check and 'X' or '', 1849 entry.cname)) 1850 else: 1851 self.putln("__Pyx_%sDECREF(%s); %s = 0;" % ( 1852 null_check and 'X' or '', 1853 self.entry_as_pyobject(entry), 1854 entry.cname)) 1855 1856 def put_var_decrefs(self, entries, used_only = 0): 1857 for entry in entries: 1858 if not used_only or entry.used: 1859 if entry.xdecref_cleanup: 1860 self.put_var_xdecref(entry) 1861 else: 1862 self.put_var_decref(entry) 1863 1864 def put_var_xdecrefs(self, entries): 1865 for entry in entries: 1866 self.put_var_xdecref(entry) 1867 1868 def put_var_xdecrefs_clear(self, entries): 1869 for entry in entries: 1870 self.put_var_xdecref_clear(entry) 1871 1872 def put_incref_memoryviewslice(self, slice_cname, have_gil=False): 1873 import MemoryView 1874 self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) 1875 self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) 1876 1877 def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False): 1878 import MemoryView 1879 self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) 1880 self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) 1881 1882 def put_xgiveref_memoryviewslice(self, slice_cname): 1883 self.put_xgiveref("%s.memview" % slice_cname) 1884 1885 def put_init_to_py_none(self, cname, type, nanny=True): 1886 from PyrexTypes import py_object_type, typecast 1887 py_none = typecast(type, py_object_type, "Py_None") 1888 if nanny: 1889 self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none)) 1890 else: 1891 self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none)) 1892 1893 def put_init_var_to_py_none(self, entry, template = "%s", nanny=True): 1894 code = template % entry.cname 1895 #if entry.type.is_extension_type: 1896 # code = "((PyObject*)%s)" % code 1897 self.put_init_to_py_none(code, entry.type, nanny) 1898 if entry.in_closure: 1899 self.put_giveref('Py_None') 1900 1901 def put_pymethoddef(self, entry, term, allow_skip=True): 1902 if entry.is_special or entry.name == '__getattribute__': 1903 if entry.name not in ['__cinit__', '__dealloc__', '__richcmp__', '__next__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', '__getcharbuffer__', '__getbuffer__', '__releasebuffer__']: 1904 if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']: 1905 pass 1906 # Python's typeobject.c will automatically fill in our slot 1907 # in add_operators() (called by PyType_Ready) with a value 1908 # that's better than ours. 1909 elif allow_skip: 1910 return 1911 from TypeSlots import method_coexist 1912 if entry.doc: 1913 doc_code = entry.doc_cname 1914 else: 1915 doc_code = 0 1916 method_flags = entry.signature.method_flags() 1917 if method_flags: 1918 if entry.is_special: 1919 method_flags += [method_coexist] 1920 self.putln( 1921 '{__Pyx_NAMESTR("%s"), (PyCFunction)%s, %s, __Pyx_DOCSTR(%s)}%s' % ( 1922 entry.name, 1923 entry.func_cname, 1924 "|".join(method_flags), 1925 doc_code, 1926 term)) 1927 1928 # GIL methods 1929 1930 def put_ensure_gil(self, declare_gilstate=True, variable=None): 1931 """ 1932 Acquire the GIL. The generated code is safe even when no PyThreadState 1933 has been allocated for this thread (for threads not initialized by 1934 using the Python API). Additionally, the code generated by this method 1935 may be called recursively. 1936 """ 1937 self.globalstate.use_utility_code( 1938 UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) 1939 self.putln("#ifdef WITH_THREAD") 1940 if not variable: 1941 variable = '__pyx_gilstate_save' 1942 if declare_gilstate: 1943 self.put("PyGILState_STATE ") 1944 self.putln("%s = PyGILState_Ensure();" % variable) 1945 self.putln("#endif") 1946 1947 def put_release_ensured_gil(self, variable=None): 1948 """ 1949 Releases the GIL, corresponds to `put_ensure_gil`. 1950 """ 1951 if not variable: 1952 variable = '__pyx_gilstate_save' 1953 self.putln("#ifdef WITH_THREAD") 1954 self.putln("PyGILState_Release(%s);" % variable) 1955 self.putln("#endif") 1956 1957 def put_acquire_gil(self, variable=None): 1958 """ 1959 Acquire the GIL. The thread's thread state must have been initialized 1960 by a previous `put_release_gil` 1961 """ 1962 self.putln("#ifdef WITH_THREAD") 1963 if variable: 1964 self.putln('_save = %s;' % variable) 1965 self.putln("Py_BLOCK_THREADS") 1966 self.putln("#endif") 1967 1968 def put_release_gil(self, variable=None): 1969 "Release the GIL, corresponds to `put_acquire_gil`." 1970 self.putln("#ifdef WITH_THREAD") 1971 self.putln("PyThreadState *_save;") 1972 self.putln("Py_UNBLOCK_THREADS") 1973 if variable: 1974 self.putln('%s = _save;' % variable) 1975 self.putln("#endif") 1976 1977 def declare_gilstate(self): 1978 self.putln("#ifdef WITH_THREAD") 1979 self.putln("PyGILState_STATE __pyx_gilstate_save;") 1980 self.putln("#endif") 1981 1982 # error handling 1983 1984 def put_error_if_neg(self, pos, value): 1985 # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos))) # TODO this path is almost _never_ taken, yet this macro makes is slower! 1986 return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos))) 1987 1988 def put_error_if_unbound(self, pos, entry, in_nogil_context=False): 1989 import ExprNodes 1990 if entry.from_closure: 1991 func = '__Pyx_RaiseClosureNameError' 1992 self.globalstate.use_utility_code( 1993 ExprNodes.raise_closure_name_error_utility_code) 1994 elif entry.type.is_memoryviewslice and in_nogil_context: 1995 func = '__Pyx_RaiseUnboundMemoryviewSliceNogil' 1996 self.globalstate.use_utility_code( 1997 ExprNodes.raise_unbound_memoryview_utility_code_nogil) 1998 else: 1999 func = '__Pyx_RaiseUnboundLocalError' 2000 self.globalstate.use_utility_code( 2001 ExprNodes.raise_unbound_local_error_utility_code) 2002 2003 self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % ( 2004 entry.type.check_for_null_code(entry.cname), 2005 func, 2006 entry.name, 2007 self.error_goto(pos))) 2008 2009 def set_error_info(self, pos, used=False): 2010 self.funcstate.should_declare_error_indicator = True 2011 if used: 2012 self.funcstate.uses_error_indicator = True 2013 if self.c_line_in_traceback: 2014 cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro) 2015 else: 2016 cinfo = "" 2017 2018 return "%s = %s[%s]; %s = %s;%s" % ( 2019 Naming.filename_cname, 2020 Naming.filetable_cname, 2021 self.lookup_filename(pos[0]), 2022 Naming.lineno_cname, 2023 pos[1], 2024 cinfo) 2025 2026 def error_goto(self, pos): 2027 lbl = self.funcstate.error_label 2028 self.funcstate.use_label(lbl) 2029 return "{%s goto %s;}" % ( 2030 self.set_error_info(pos), 2031 lbl) 2032 2033 def error_goto_if(self, cond, pos): 2034 return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos)) 2035 2036 def error_goto_if_null(self, cname, pos): 2037 return self.error_goto_if("!%s" % cname, pos) 2038 2039 def error_goto_if_neg(self, cname, pos): 2040 return self.error_goto_if("%s < 0" % cname, pos) 2041 2042 def error_goto_if_PyErr(self, pos): 2043 return self.error_goto_if("PyErr_Occurred()", pos) 2044 2045 def lookup_filename(self, filename): 2046 return self.globalstate.lookup_filename(filename) 2047 2048 def put_declare_refcount_context(self): 2049 self.putln('__Pyx_RefNannyDeclarations') 2050 2051 def put_setup_refcount_context(self, name, acquire_gil=False): 2052 if acquire_gil: 2053 self.globalstate.use_utility_code( 2054 UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) 2055 self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0)) 2056 2057 def put_finish_refcount_context(self): 2058 self.putln("__Pyx_RefNannyFinishContext();") 2059 2060 def put_add_traceback(self, qualified_name): 2061 """ 2062 Build a Python traceback for propagating exceptions. 2063 2064 qualified_name should be the qualified name of the function. 2065 """ 2066 format_tuple = ( 2067 qualified_name, 2068 Naming.clineno_cname, 2069 Naming.lineno_cname, 2070 Naming.filename_cname, 2071 ) 2072 self.funcstate.uses_error_indicator = True 2073 self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple) 2074 2075 def put_unraisable(self, qualified_name): 2076 """ 2077 Generate code to print a Python warning for an unraisable exception. 2078 2079 qualified_name should be the qualified name of the function. 2080 """ 2081 format_tuple = ( 2082 qualified_name, 2083 Naming.clineno_cname, 2084 Naming.lineno_cname, 2085 Naming.filename_cname, 2086 int(self.globalstate.directives['unraisable_tracebacks']) 2087 ) 2088 self.funcstate.uses_error_indicator = True 2089 self.putln('__Pyx_WriteUnraisable("%s", %s, %s, %s, %s);' % format_tuple) 2090 self.globalstate.use_utility_code( 2091 UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c")) 2092 2093 def put_trace_declarations(self): 2094 self.putln('__Pyx_TraceDeclarations') 2095 2096 def put_trace_call(self, name, pos): 2097 self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1])) 2098 2099 def put_trace_exception(self): 2100 self.putln("__Pyx_TraceException();") 2101 2102 def put_trace_return(self, retvalue_cname): 2103 self.putln("__Pyx_TraceReturn(%s);" % retvalue_cname) 2104 2105 def putln_openmp(self, string): 2106 self.putln("#ifdef _OPENMP") 2107 self.putln(string) 2108 self.putln("#endif /* _OPENMP */") 2109 2110 def undef_builtin_expect(self, cond): 2111 """ 2112 Redefine the macros likely() and unlikely to no-ops, depending on 2113 condition 'cond' 2114 """ 2115 self.putln("#if %s" % cond) 2116 self.putln(" #undef likely") 2117 self.putln(" #undef unlikely") 2118 self.putln(" #define likely(x) (x)") 2119 self.putln(" #define unlikely(x) (x)") 2120 self.putln("#endif") 2121 2122 def redef_builtin_expect(self, cond): 2123 self.putln("#if %s" % cond) 2124 self.putln(" #undef likely") 2125 self.putln(" #undef unlikely") 2126 self.putln(" #define likely(x) __builtin_expect(!!(x), 1)") 2127 self.putln(" #define unlikely(x) __builtin_expect(!!(x), 0)") 2128 self.putln("#endif") 2129 2130 class PyrexCodeWriter(object): 2131 # f file output file 2132 # level int indentation level 2133 2134 def __init__(self, outfile_name): 2135 self.f = Utils.open_new_file(outfile_name) 2136 self.level = 0 2137 2138 def putln(self, code): 2139 self.f.write("%s%s\n" % (" " * self.level, code)) 2140 2141 def indent(self): 2142 self.level += 1 2143 2144 def dedent(self): 2145 self.level -= 1 2146 2147 class PyxCodeWriter(object): 2148 """ 2149 Can be used for writing out some Cython code. To use the indenter 2150 functionality, the Cython.Compiler.Importer module will have to be used 2151 to load the code to support python 2.4 2152 """ 2153 2154 def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'): 2155 self.buffer = buffer or StringIOTree() 2156 self.level = indent_level 2157 self.context = context 2158 self.encoding = encoding 2159 2160 def indent(self, levels=1): 2161 self.level += levels 2162 return True 2163 2164 def dedent(self, levels=1): 2165 self.level -= levels 2166 2167 def indenter(self, line): 2168 """ 2169 Instead of 2170 2171 with pyx_code.indenter("for i in range(10):"): 2172 pyx_code.putln("print i") 2173 2174 write 2175 2176 if pyx_code.indenter("for i in range(10);"): 2177 pyx_code.putln("print i") 2178 pyx_code.dedent() 2179 """ 2180 self.putln(line) 2181 self.indent() 2182 return True 2183 2184 def getvalue(self): 2185 result = self.buffer.getvalue() 2186 if not isinstance(result, unicode): 2187 result = result.decode(self.encoding) 2188 2189 return result 2190 2191 def putln(self, line, context=None): 2192 context = context or self.context 2193 if context: 2194 line = sub_tempita(line, context) 2195 self._putln(line) 2196 2197 def _putln(self, line): 2198 self.buffer.write("%s%s\n" % (self.level * " ", line)) 2199 2200 def put_chunk(self, chunk, context=None): 2201 context = context or self.context 2202 if context: 2203 chunk = sub_tempita(chunk, context) 2204 2205 chunk = textwrap.dedent(chunk) 2206 for line in chunk.splitlines(): 2207 self._putln(line) 2208 2209 def insertion_point(self): 2210 return PyxCodeWriter(self.buffer.insertion_point(), self.level, 2211 self.context) 2212 2213 def named_insertion_point(self, name): 2214 setattr(self, name, self.insertion_point()) 2215 2216 2217 class ClosureTempAllocator(object): 2218 def __init__(self, klass): 2219 self.klass = klass 2220 self.temps_allocated = {} 2221 self.temps_free = {} 2222 self.temps_count = 0 2223 2224 def reset(self): 2225 for type, cnames in self.temps_allocated.items(): 2226 self.temps_free[type] = list(cnames) 2227 2228 def allocate_temp(self, type): 2229 if not type in self.temps_allocated: 2230 self.temps_allocated[type] = [] 2231 self.temps_free[type] = [] 2232 elif self.temps_free[type]: 2233 return self.temps_free[type].pop(0) 2234 cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count) 2235 self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True) 2236 self.temps_allocated[type].append(cname) 2237 self.temps_count += 1 2238 return cname 2239
