1 import itertools 2 from time import time 3 4 import Errors 5 import DebugFlags 6 import Options 7 from Visitor import CythonTransform 8 from Errors import CompileError, InternalError, AbortError 9 import Naming 10 11 # 12 # Really small pipeline stages 13 # 14 def dumptree(t): 15 # For quick debugging in pipelines 16 print t.dump() 17 return t 18 19 def abort_on_errors(node): 20 # Stop the pipeline if there are any errors. 21 if Errors.num_errors != 0: 22 raise AbortError("pipeline break") 23 return node 24 25 def parse_stage_factory(context): 26 def parse(compsrc): 27 source_desc = compsrc.source_desc 28 full_module_name = compsrc.full_module_name 29 initial_pos = (source_desc, 1, 0) 30 saved_cimport_from_pyx, Options.cimport_from_pyx = Options.cimport_from_pyx, False 31 scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0, 32 check_module_name = not Options.embed) 33 Options.cimport_from_pyx = saved_cimport_from_pyx 34 tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name) 35 tree.compilation_source = compsrc 36 tree.scope = scope 37 tree.is_pxd = False 38 return tree 39 return parse 40 41 def parse_pxd_stage_factory(context, scope, module_name): 42 def parse(source_desc): 43 tree = context.parse(source_desc, scope, pxd=True, 44 full_module_name=module_name) 45 tree.scope = scope 46 tree.is_pxd = True 47 return tree 48 return parse 49 50 def generate_pyx_code_stage_factory(options, result): 51 def generate_pyx_code_stage(module_node): 52 module_node.process_implementation(options, result) 53 result.compilation_source = module_node.compilation_source 54 return result 55 return generate_pyx_code_stage 56 57 def inject_pxd_code_stage_factory(context): 58 def inject_pxd_code_stage(module_node): 59 from textwrap import dedent 60 stats = module_node.body.stats 61 for name, (statlistnode, scope) in context.pxds.iteritems(): 62 module_node.merge_in(statlistnode, scope) 63 return module_node 64 return inject_pxd_code_stage 65 66 def use_utility_code_definitions(scope, target, seen=None): 67 if seen is None: 68 seen = set() 69 70 for entry in scope.entries.itervalues(): 71 if entry in seen: 72 continue 73 74 seen.add(entry) 75 if entry.used and entry.utility_code_definition: 76 target.use_utility_code(entry.utility_code_definition) 77 for required_utility in entry.utility_code_definition.requires: 78 target.use_utility_code(required_utility) 79 elif entry.as_module: 80 use_utility_code_definitions(entry.as_module, target, seen) 81 82 def inject_utility_code_stage_factory(context): 83 def inject_utility_code_stage(module_node): 84 use_utility_code_definitions(context.cython_scope, module_node.scope) 85 added = [] 86 # Note: the list might be extended inside the loop (if some utility code 87 # pulls in other utility code, explicitly or implicitly) 88 for utilcode in module_node.scope.utility_code_list: 89 if utilcode in added: continue 90 added.append(utilcode) 91 if utilcode.requires: 92 for dep in utilcode.requires: 93 if not dep in added and not dep in module_node.scope.utility_code_list: 94 module_node.scope.utility_code_list.append(dep) 95 tree = utilcode.get_tree() 96 if tree: 97 module_node.merge_in(tree.body, tree.scope, merge_scope=True) 98 return module_node 99 return inject_utility_code_stage 100 101 class UseUtilityCodeDefinitions(CythonTransform): 102 # Temporary hack to use any utility code in nodes' "utility_code_definitions". 103 # This should be moved to the code generation phase of the relevant nodes once 104 # it is safe to generate CythonUtilityCode at code generation time. 105 def __call__(self, node): 106 self.scope = node.scope 107 return super(UseUtilityCodeDefinitions, self).__call__(node) 108 109 def process_entry(self, entry): 110 if entry: 111 for utility_code in (entry.utility_code, entry.utility_code_definition): 112 if utility_code: 113 self.scope.use_utility_code(utility_code) 114 115 def visit_AttributeNode(self, node): 116 self.process_entry(node.entry) 117 return node 118 119 def visit_NameNode(self, node): 120 self.process_entry(node.entry) 121 self.process_entry(node.type_entry) 122 return node 123 124 # 125 # Pipeline factories 126 # 127 128 def create_pipeline(context, mode, exclude_classes=()): 129 assert mode in ('pyx', 'py', 'pxd') 130 from Visitor import PrintTree 131 from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse 132 from ParseTreeTransforms import ForwardDeclareTypes, AnalyseDeclarationsTransform 133 from ParseTreeTransforms import AnalyseExpressionsTransform, FindInvalidUseOfFusedTypes 134 from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform 135 from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods 136 from ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform 137 from ParseTreeTransforms import CalculateQualifiedNamesTransform 138 from TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic 139 from ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions 140 from ParseTreeTransforms import RemoveUnreachableCode, GilCheck 141 from FlowControl import ControlFlowAnalysis 142 from AnalysedTreeTransforms import AutoTestDictTransform 143 from AutoDocTransforms import EmbedSignature 144 from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform 145 from Optimize import EarlyReplaceBuiltinCalls, OptimizeBuiltinCalls 146 from Optimize import InlineDefNodeCalls 147 from Optimize import ConstantFolding, FinalOptimizePhase 148 from Optimize import DropRefcountingTransform 149 from Optimize import ConsolidateOverflowCheck 150 from Buffer import IntroduceBufferAuxiliaryVars 151 from ModuleNode import check_c_declarations, check_c_declarations_pxd 152 153 154 if mode == 'pxd': 155 _check_c_declarations = check_c_declarations_pxd 156 _specific_post_parse = PxdPostParse(context) 157 else: 158 _check_c_declarations = check_c_declarations 159 _specific_post_parse = None 160 161 if mode == 'py': 162 _align_function_definitions = AlignFunctionDefinitions(context) 163 else: 164 _align_function_definitions = None 165 166 # NOTE: This is the "common" parts of the pipeline, which is also 167 # code in pxd files. So it will be run multiple times in a 168 # compilation stage. 169 stages = [ 170 NormalizeTree(context), 171 PostParse(context), 172 _specific_post_parse, 173 InterpretCompilerDirectives(context, context.compiler_directives), 174 ParallelRangeTransform(context), 175 AdjustDefByDirectives(context), 176 MarkClosureVisitor(context), 177 _align_function_definitions, 178 RemoveUnreachableCode(context), 179 ConstantFolding(), 180 FlattenInListTransform(), 181 WithTransform(context), 182 DecoratorTransform(context), 183 ForwardDeclareTypes(context), 184 AnalyseDeclarationsTransform(context), 185 AutoTestDictTransform(context), 186 EmbedSignature(context), 187 EarlyReplaceBuiltinCalls(context), ## Necessary? 188 TransformBuiltinMethods(context), ## Necessary? 189 MarkParallelAssignments(context), 190 ControlFlowAnalysis(context), 191 RemoveUnreachableCode(context), 192 # MarkParallelAssignments(context), 193 MarkOverflowingArithmetic(context), 194 IntroduceBufferAuxiliaryVars(context), 195 _check_c_declarations, 196 InlineDefNodeCalls(context), 197 AnalyseExpressionsTransform(context), 198 FindInvalidUseOfFusedTypes(context), 199 ExpandInplaceOperators(context), 200 OptimizeBuiltinCalls(context), ## Necessary? 201 CreateClosureClasses(context), ## After all lookups and type inference 202 CalculateQualifiedNamesTransform(context), 203 ConsolidateOverflowCheck(context), 204 IterationTransform(context), 205 SwitchTransform(), 206 DropRefcountingTransform(), 207 FinalOptimizePhase(context), 208 GilCheck(), 209 UseUtilityCodeDefinitions(context), 210 ] 211 filtered_stages = [] 212 for s in stages: 213 if s.__class__ not in exclude_classes: 214 filtered_stages.append(s) 215 return filtered_stages 216 217 def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()): 218 if py: 219 mode = 'py' 220 else: 221 mode = 'pyx' 222 test_support = [] 223 if options.evaluate_tree_assertions: 224 from Cython.TestUtils import TreeAssertVisitor 225 test_support.append(TreeAssertVisitor()) 226 227 if options.gdb_debug: 228 from Cython.Debugger import DebugWriter # requires Py2.5+ 229 from ParseTreeTransforms import DebugTransform 230 context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter( 231 options.output_dir) 232 debug_transform = [DebugTransform(context, options, result)] 233 else: 234 debug_transform = [] 235 236 return list(itertools.chain( 237 [parse_stage_factory(context)], 238 create_pipeline(context, mode, exclude_classes=exclude_classes), 239 test_support, 240 [inject_pxd_code_stage_factory(context), 241 inject_utility_code_stage_factory(context), 242 abort_on_errors], 243 debug_transform, 244 [generate_pyx_code_stage_factory(options, result)])) 245 246 def create_pxd_pipeline(context, scope, module_name): 247 from CodeGeneration import ExtractPxdCode 248 249 # The pxd pipeline ends up with a CCodeWriter containing the 250 # code of the pxd, as well as a pxd scope. 251 return [ 252 parse_pxd_stage_factory(context, scope, module_name) 253 ] + create_pipeline(context, 'pxd') + [ 254 ExtractPxdCode() 255 ] 256 257 def create_py_pipeline(context, options, result): 258 return create_pyx_pipeline(context, options, result, py=True) 259 260 def create_pyx_as_pxd_pipeline(context, result): 261 from ParseTreeTransforms import AlignFunctionDefinitions, \ 262 MarkClosureVisitor, WithTransform, AnalyseDeclarationsTransform 263 from Optimize import ConstantFolding, FlattenInListTransform 264 from Nodes import StatListNode 265 pipeline = [] 266 pyx_pipeline = create_pyx_pipeline(context, context.options, result, 267 exclude_classes=[ 268 AlignFunctionDefinitions, 269 MarkClosureVisitor, 270 ConstantFolding, 271 FlattenInListTransform, 272 WithTransform 273 ]) 274 for stage in pyx_pipeline: 275 pipeline.append(stage) 276 if isinstance(stage, AnalyseDeclarationsTransform): 277 # This is the last stage we need. 278 break 279 def fake_pxd(root): 280 for entry in root.scope.entries.values(): 281 if not entry.in_cinclude: 282 entry.defined_in_pxd = 1 283 if entry.name == entry.cname and entry.visibility != 'extern': 284 # Always mangle non-extern cimported entries. 285 entry.cname = entry.scope.mangle(Naming.func_prefix, entry.name) 286 return StatListNode(root.pos, stats=[]), root.scope 287 pipeline.append(fake_pxd) 288 return pipeline 289 290 def insert_into_pipeline(pipeline, transform, before=None, after=None): 291 """ 292 Insert a new transform into the pipeline after or before an instance of 293 the given class. e.g. 294 295 pipeline = insert_into_pipeline(pipeline, transform, 296 after=AnalyseDeclarationsTransform) 297 """ 298 assert before or after 299 300 cls = before or after 301 for i, t in enumerate(pipeline): 302 if isinstance(t, cls): 303 break 304 305 if after: 306 i += 1 307 308 return pipeline[:i] + [transform] + pipeline[i:] 309 310 # 311 # Running a pipeline 312 # 313 314 def run_pipeline(pipeline, source, printtree=True): 315 from Cython.Compiler.Visitor import PrintTree 316 317 error = None 318 data = source 319 try: 320 try: 321 for phase in pipeline: 322 if phase is not None: 323 if DebugFlags.debug_verbose_pipeline: 324 t = time() 325 print "Entering pipeline phase %r" % phase 326 if not printtree and isinstance(phase, PrintTree): 327 continue 328 data = phase(data) 329 if DebugFlags.debug_verbose_pipeline: 330 print " %.3f seconds" % (time() - t) 331 except CompileError, err: 332 # err is set 333 Errors.report_error(err) 334 error = err 335 except InternalError, err: 336 # Only raise if there was not an earlier error 337 if Errors.num_errors == 0: 338 raise 339 error = err 340 except AbortError, err: 341 error = err 342 return (error, data) 343
