1 # -*- coding: utf-8 -*- 2 """ 3 jinja2.ext 4 ~~~~~~~~~~ 5 6 Jinja extensions allow to add custom tags similar to the way django custom 7 tags work. By default two example extensions exist: an i18n and a cache 8 extension. 9 10 :copyright: (c) 2010 by the Jinja Team. 11 :license: BSD. 12 """ 13 from jinja2 import nodes 14 from jinja2.defaults import BLOCK_START_STRING, \ 15 BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ 16 COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ 17 LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ 18 KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS 19 from jinja2.environment import Environment 20 from jinja2.runtime import concat 21 from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError 22 from jinja2.utils import contextfunction, import_string, Markup 23 from jinja2._compat import next, with_metaclass, string_types, iteritems 24 25 26 # the only real useful gettext functions for a Jinja template. Note 27 # that ugettext must be assigned to gettext as Jinja doesn't support 28 # non unicode strings. 29 GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext') 30 31 32 class ExtensionRegistry(type): 33 """Gives the extension an unique identifier.""" 34 35 def __new__(cls, name, bases, d): 36 rv = type.__new__(cls, name, bases, d) 37 rv.identifier = rv.__module__ + '.' + rv.__name__ 38 return rv 39 40 41 class Extension(with_metaclass(ExtensionRegistry, object)): 42 """Extensions can be used to add extra functionality to the Jinja template 43 system at the parser level. Custom extensions are bound to an environment 44 but may not store environment specific data on `self`. The reason for 45 this is that an extension can be bound to another environment (for 46 overlays) by creating a copy and reassigning the `environment` attribute. 47 48 As extensions are created by the environment they cannot accept any 49 arguments for configuration. One may want to work around that by using 50 a factory function, but that is not possible as extensions are identified 51 by their import name. The correct way to configure the extension is 52 storing the configuration values on the environment. Because this way the 53 environment ends up acting as central configuration storage the 54 attributes may clash which is why extensions have to ensure that the names 55 they choose for configuration are not too generic. ``prefix`` for example 56 is a terrible name, ``fragment_cache_prefix`` on the other hand is a good 57 name as includes the name of the extension (fragment cache). 58 """ 59 60 #: if this extension parses this is the list of tags it's listening to. 61 tags = set() 62 63 #: the priority of that extension. This is especially useful for 64 #: extensions that preprocess values. A lower value means higher 65 #: priority. 66 #: 67 #: .. versionadded:: 2.4 68 priority = 100 69 70 def __init__(self, environment): 71 self.environment = environment 72 73 def bind(self, environment): 74 """Create a copy of this extension bound to another environment.""" 75 rv = object.__new__(self.__class__) 76 rv.__dict__.update(self.__dict__) 77 rv.environment = environment 78 return rv 79 80 def preprocess(self, source, name, filename=None): 81 """This method is called before the actual lexing and can be used to 82 preprocess the source. The `filename` is optional. The return value 83 must be the preprocessed source. 84 """ 85 return source 86 87 def filter_stream(self, stream): 88 """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used 89 to filter tokens returned. This method has to return an iterable of 90 :class:`~jinja2.lexer.Token`\s, but it doesn't have to return a 91 :class:`~jinja2.lexer.TokenStream`. 92 93 In the `ext` folder of the Jinja2 source distribution there is a file 94 called `inlinegettext.py` which implements a filter that utilizes this 95 method. 96 """ 97 return stream 98 99 def parse(self, parser): 100 """If any of the :attr:`tags` matched this method is called with the 101 parser as first argument. The token the parser stream is pointing at 102 is the name token that matched. This method has to return one or a 103 list of multiple nodes. 104 """ 105 raise NotImplementedError() 106 107 def attr(self, name, lineno=None): 108 """Return an attribute node for the current extension. This is useful 109 to pass constants on extensions to generated template code. 110 111 :: 112 113 self.attr('_my_attribute', lineno=lineno) 114 """ 115 return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno) 116 117 def call_method(self, name, args=None, kwargs=None, dyn_args=None, 118 dyn_kwargs=None, lineno=None): 119 """Call a method of the extension. This is a shortcut for 120 :meth:`attr` + :class:`jinja2.nodes.Call`. 121 """ 122 if args is None: 123 args = [] 124 if kwargs is None: 125 kwargs = [] 126 return nodes.Call(self.attr(name, lineno=lineno), args, kwargs, 127 dyn_args, dyn_kwargs, lineno=lineno) 128 129 130 @contextfunction 131 def _gettext_alias(__context, *args, **kwargs): 132 return __context.call(__context.resolve('gettext'), *args, **kwargs) 133 134 135 def _make_new_gettext(func): 136 @contextfunction 137 def gettext(__context, __string, **variables): 138 rv = __context.call(func, __string) 139 if __context.eval_ctx.autoescape: 140 rv = Markup(rv) 141 return rv % variables 142 return gettext 143 144 145 def _make_new_ngettext(func): 146 @contextfunction 147 def ngettext(__context, __singular, __plural, __num, **variables): 148 variables.setdefault('num', __num) 149 rv = __context.call(func, __singular, __plural, __num) 150 if __context.eval_ctx.autoescape: 151 rv = Markup(rv) 152 return rv % variables 153 return ngettext 154 155 156 class InternationalizationExtension(Extension): 157 """This extension adds gettext support to Jinja2.""" 158 tags = set(['trans']) 159 160 # TODO: the i18n extension is currently reevaluating values in a few 161 # situations. Take this example: 162 # {% trans count=something() %}{{ count }} foo{% pluralize 163 # %}{{ count }} fooss{% endtrans %} 164 # something is called twice here. One time for the gettext value and 165 # the other time for the n-parameter of the ngettext function. 166 167 def __init__(self, environment): 168 Extension.__init__(self, environment) 169 environment.globals['_'] = _gettext_alias 170 environment.extend( 171 install_gettext_translations=self._install, 172 install_null_translations=self._install_null, 173 install_gettext_callables=self._install_callables, 174 uninstall_gettext_translations=self._uninstall, 175 extract_translations=self._extract, 176 newstyle_gettext=False 177 ) 178 179 def _install(self, translations, newstyle=None): 180 gettext = getattr(translations, 'ugettext', None) 181 if gettext is None: 182 gettext = translations.gettext 183 ngettext = getattr(translations, 'ungettext', None) 184 if ngettext is None: 185 ngettext = translations.ngettext 186 self._install_callables(gettext, ngettext, newstyle) 187 188 def _install_null(self, newstyle=None): 189 self._install_callables( 190 lambda x: x, 191 lambda s, p, n: (n != 1 and (p,) or (s,))[0], 192 newstyle 193 ) 194 195 def _install_callables(self, gettext, ngettext, newstyle=None): 196 if newstyle is not None: 197 self.environment.newstyle_gettext = newstyle 198 if self.environment.newstyle_gettext: 199 gettext = _make_new_gettext(gettext) 200 ngettext = _make_new_ngettext(ngettext) 201 self.environment.globals.update( 202 gettext=gettext, 203 ngettext=ngettext 204 ) 205 206 def _uninstall(self, translations): 207 for key in 'gettext', 'ngettext': 208 self.environment.globals.pop(key, None) 209 210 def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS): 211 if isinstance(source, string_types): 212 source = self.environment.parse(source) 213 return extract_from_ast(source, gettext_functions) 214 215 def parse(self, parser): 216 """Parse a translatable tag.""" 217 lineno = next(parser.stream).lineno 218 num_called_num = False 219 220 # find all the variables referenced. Additionally a variable can be 221 # defined in the body of the trans block too, but this is checked at 222 # a later state. 223 plural_expr = None 224 plural_expr_assignment = None 225 variables = {} 226 while parser.stream.current.type != 'block_end': 227 if variables: 228 parser.stream.expect('comma') 229 230 # skip colon for python compatibility 231 if parser.stream.skip_if('colon'): 232 break 233 234 name = parser.stream.expect('name') 235 if name.value in variables: 236 parser.fail('translatable variable %r defined twice.' % 237 name.value, name.lineno, 238 exc=TemplateAssertionError) 239 240 # expressions 241 if parser.stream.current.type == 'assign': 242 next(parser.stream) 243 variables[name.value] = var = parser.parse_expression() 244 else: 245 variables[name.value] = var = nodes.Name(name.value, 'load') 246 247 if plural_expr is None: 248 if isinstance(var, nodes.Call): 249 plural_expr = nodes.Name('_trans', 'load') 250 variables[name.value] = plural_expr 251 plural_expr_assignment = nodes.Assign( 252 nodes.Name('_trans', 'store'), var) 253 else: 254 plural_expr = var 255 num_called_num = name.value == 'num' 256 257 parser.stream.expect('block_end') 258 259 plural = plural_names = None 260 have_plural = False 261 referenced = set() 262 263 # now parse until endtrans or pluralize 264 singular_names, singular = self._parse_block(parser, True) 265 if singular_names: 266 referenced.update(singular_names) 267 if plural_expr is None: 268 plural_expr = nodes.Name(singular_names[0], 'load') 269 num_called_num = singular_names[0] == 'num' 270 271 # if we have a pluralize block, we parse that too 272 if parser.stream.current.test('name:pluralize'): 273 have_plural = True 274 next(parser.stream) 275 if parser.stream.current.type != 'block_end': 276 name = parser.stream.expect('name') 277 if name.value not in variables: 278 parser.fail('unknown variable %r for pluralization' % 279 name.value, name.lineno, 280 exc=TemplateAssertionError) 281 plural_expr = variables[name.value] 282 num_called_num = name.value == 'num' 283 parser.stream.expect('block_end') 284 plural_names, plural = self._parse_block(parser, False) 285 next(parser.stream) 286 referenced.update(plural_names) 287 else: 288 next(parser.stream) 289 290 # register free names as simple name expressions 291 for var in referenced: 292 if var not in variables: 293 variables[var] = nodes.Name(var, 'load') 294 295 if not have_plural: 296 plural_expr = None 297 elif plural_expr is None: 298 parser.fail('pluralize without variables', lineno) 299 300 node = self._make_node(singular, plural, variables, plural_expr, 301 bool(referenced), 302 num_called_num and have_plural) 303 node.set_lineno(lineno) 304 if plural_expr_assignment is not None: 305 return [plural_expr_assignment, node] 306 else: 307 return node 308 309 def _parse_block(self, parser, allow_pluralize): 310 """Parse until the next block tag with a given name.""" 311 referenced = [] 312 buf = [] 313 while 1: 314 if parser.stream.current.type == 'data': 315 buf.append(parser.stream.current.value.replace('%', '%%')) 316 next(parser.stream) 317 elif parser.stream.current.type == 'variable_begin': 318 next(parser.stream) 319 name = parser.stream.expect('name').value 320 referenced.append(name) 321 buf.append('%%(%s)s' % name) 322 parser.stream.expect('variable_end') 323 elif parser.stream.current.type == 'block_begin': 324 next(parser.stream) 325 if parser.stream.current.test('name:endtrans'): 326 break 327 elif parser.stream.current.test('name:pluralize'): 328 if allow_pluralize: 329 break 330 parser.fail('a translatable section can have only one ' 331 'pluralize section') 332 parser.fail('control structures in translatable sections are ' 333 'not allowed') 334 elif parser.stream.eos: 335 parser.fail('unclosed translation block') 336 else: 337 assert False, 'internal parser error' 338 339 return referenced, concat(buf) 340 341 def _make_node(self, singular, plural, variables, plural_expr, 342 vars_referenced, num_called_num): 343 """Generates a useful node from the data provided.""" 344 # no variables referenced? no need to escape for old style 345 # gettext invocations only if there are vars. 346 if not vars_referenced and not self.environment.newstyle_gettext: 347 singular = singular.replace('%%', '%') 348 if plural: 349 plural = plural.replace('%%', '%') 350 351 # singular only: 352 if plural_expr is None: 353 gettext = nodes.Name('gettext', 'load') 354 node = nodes.Call(gettext, [nodes.Const(singular)], 355 [], None, None) 356 357 # singular and plural 358 else: 359 ngettext = nodes.Name('ngettext', 'load') 360 node = nodes.Call(ngettext, [ 361 nodes.Const(singular), 362 nodes.Const(plural), 363 plural_expr 364 ], [], None, None) 365 366 # in case newstyle gettext is used, the method is powerful 367 # enough to handle the variable expansion and autoescape 368 # handling itself 369 if self.environment.newstyle_gettext: 370 for key, value in iteritems(variables): 371 # the function adds that later anyways in case num was 372 # called num, so just skip it. 373 if num_called_num and key == 'num': 374 continue 375 node.kwargs.append(nodes.Keyword(key, value)) 376 377 # otherwise do that here 378 else: 379 # mark the return value as safe if we are in an 380 # environment with autoescaping turned on 381 node = nodes.MarkSafeIfAutoescape(node) 382 if variables: 383 node = nodes.Mod(node, nodes.Dict([ 384 nodes.Pair(nodes.Const(key), value) 385 for key, value in variables.items() 386 ])) 387 return nodes.Output([node]) 388 389 390 class ExprStmtExtension(Extension): 391 """Adds a `do` tag to Jinja2 that works like the print statement just 392 that it doesn't print the return value. 393 """ 394 tags = set(['do']) 395 396 def parse(self, parser): 397 node = nodes.ExprStmt(lineno=next(parser.stream).lineno) 398 node.node = parser.parse_tuple() 399 return node 400 401 402 class LoopControlExtension(Extension): 403 """Adds break and continue to the template engine.""" 404 tags = set(['break', 'continue']) 405 406 def parse(self, parser): 407 token = next(parser.stream) 408 if token.value == 'break': 409 return nodes.Break(lineno=token.lineno) 410 return nodes.Continue(lineno=token.lineno) 411 412 413 class WithExtension(Extension): 414 """Adds support for a django-like with block.""" 415 tags = set(['with']) 416 417 def parse(self, parser): 418 node = nodes.Scope(lineno=next(parser.stream).lineno) 419 assignments = [] 420 while parser.stream.current.type != 'block_end': 421 lineno = parser.stream.current.lineno 422 if assignments: 423 parser.stream.expect('comma') 424 target = parser.parse_assign_target() 425 parser.stream.expect('assign') 426 expr = parser.parse_expression() 427 assignments.append(nodes.Assign(target, expr, lineno=lineno)) 428 node.body = assignments + \ 429 list(parser.parse_statements(('name:endwith',), 430 drop_needle=True)) 431 return node 432 433 434 class AutoEscapeExtension(Extension): 435 """Changes auto escape rules for a scope.""" 436 tags = set(['autoescape']) 437 438 def parse(self, parser): 439 node = nodes.ScopedEvalContextModifier(lineno=next(parser.stream).lineno) 440 node.options = [ 441 nodes.Keyword('autoescape', parser.parse_expression()) 442 ] 443 node.body = parser.parse_statements(('name:endautoescape',), 444 drop_needle=True) 445 return nodes.Scope([node]) 446 447 448 def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, 449 babel_style=True): 450 """Extract localizable strings from the given template node. Per 451 default this function returns matches in babel style that means non string 452 parameters as well as keyword arguments are returned as `None`. This 453 allows Babel to figure out what you really meant if you are using 454 gettext functions that allow keyword arguments for placeholder expansion. 455 If you don't want that behavior set the `babel_style` parameter to `False` 456 which causes only strings to be returned and parameters are always stored 457 in tuples. As a consequence invalid gettext calls (calls without a single 458 string parameter or string parameters after non-string parameters) are 459 skipped. 460 461 This example explains the behavior: 462 463 >>> from jinja2 import Environment 464 >>> env = Environment() 465 >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}') 466 >>> list(extract_from_ast(node)) 467 [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))] 468 >>> list(extract_from_ast(node, babel_style=False)) 469 [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))] 470 471 For every string found this function yields a ``(lineno, function, 472 message)`` tuple, where: 473 474 * ``lineno`` is the number of the line on which the string was found, 475 * ``function`` is the name of the ``gettext`` function used (if the 476 string was extracted from embedded Python code), and 477 * ``message`` is the string itself (a ``unicode`` object, or a tuple 478 of ``unicode`` objects for functions with multiple string arguments). 479 480 This extraction function operates on the AST and is because of that unable 481 to extract any comments. For comment support you have to use the babel 482 extraction interface or extract comments yourself. 483 """ 484 for node in node.find_all(nodes.Call): 485 if not isinstance(node.node, nodes.Name) or \ 486 node.node.name not in gettext_functions: 487 continue 488 489 strings = [] 490 for arg in node.args: 491 if isinstance(arg, nodes.Const) and \ 492 isinstance(arg.value, string_types): 493 strings.append(arg.value) 494 else: 495 strings.append(None) 496 497 for arg in node.kwargs: 498 strings.append(None) 499 if node.dyn_args is not None: 500 strings.append(None) 501 if node.dyn_kwargs is not None: 502 strings.append(None) 503 504 if not babel_style: 505 strings = tuple(x for x in strings if x is not None) 506 if not strings: 507 continue 508 else: 509 if len(strings) == 1: 510 strings = strings[0] 511 else: 512 strings = tuple(strings) 513 yield node.lineno, node.node.name, strings 514 515 516 class _CommentFinder(object): 517 """Helper class to find comments in a token stream. Can only 518 find comments for gettext calls forwards. Once the comment 519 from line 4 is found, a comment for line 1 will not return a 520 usable value. 521 """ 522 523 def __init__(self, tokens, comment_tags): 524 self.tokens = tokens 525 self.comment_tags = comment_tags 526 self.offset = 0 527 self.last_lineno = 0 528 529 def find_backwards(self, offset): 530 try: 531 for _, token_type, token_value in \ 532 reversed(self.tokens[self.offset:offset]): 533 if token_type in ('comment', 'linecomment'): 534 try: 535 prefix, comment = token_value.split(None, 1) 536 except ValueError: 537 continue 538 if prefix in self.comment_tags: 539 return [comment.rstrip()] 540 return [] 541 finally: 542 self.offset = offset 543 544 def find_comments(self, lineno): 545 if not self.comment_tags or self.last_lineno > lineno: 546 return [] 547 for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]): 548 if token_lineno > lineno: 549 return self.find_backwards(self.offset + idx) 550 return self.find_backwards(len(self.tokens)) 551 552 553 def babel_extract(fileobj, keywords, comment_tags, options): 554 """Babel extraction method for Jinja templates. 555 556 .. versionchanged:: 2.3 557 Basic support for translation comments was added. If `comment_tags` 558 is now set to a list of keywords for extraction, the extractor will 559 try to find the best preceeding comment that begins with one of the 560 keywords. For best results, make sure to not have more than one 561 gettext call in one line of code and the matching comment in the 562 same line or the line before. 563 564 .. versionchanged:: 2.5.1 565 The `newstyle_gettext` flag can be set to `True` to enable newstyle 566 gettext calls. 567 568 .. versionchanged:: 2.7 569 A `silent` option can now be provided. If set to `False` template 570 syntax errors are propagated instead of being ignored. 571 572 :param fileobj: the file-like object the messages should be extracted from 573 :param keywords: a list of keywords (i.e. function names) that should be 574 recognized as translation functions 575 :param comment_tags: a list of translator tags to search for and include 576 in the results. 577 :param options: a dictionary of additional options (optional) 578 :return: an iterator over ``(lineno, funcname, message, comments)`` tuples. 579 (comments will be empty currently) 580 """ 581 extensions = set() 582 for extension in options.get('extensions', '').split(','): 583 extension = extension.strip() 584 if not extension: 585 continue 586 extensions.add(import_string(extension)) 587 if InternationalizationExtension not in extensions: 588 extensions.add(InternationalizationExtension) 589 590 def getbool(options, key, default=False): 591 return options.get(key, str(default)).lower() in \ 592 ('1', 'on', 'yes', 'true') 593 594 silent = getbool(options, 'silent', True) 595 environment = Environment( 596 options.get('block_start_string', BLOCK_START_STRING), 597 options.get('block_end_string', BLOCK_END_STRING), 598 options.get('variable_start_string', VARIABLE_START_STRING), 599 options.get('variable_end_string', VARIABLE_END_STRING), 600 options.get('comment_start_string', COMMENT_START_STRING), 601 options.get('comment_end_string', COMMENT_END_STRING), 602 options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX, 603 options.get('line_comment_prefix') or LINE_COMMENT_PREFIX, 604 getbool(options, 'trim_blocks', TRIM_BLOCKS), 605 getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS), 606 NEWLINE_SEQUENCE, 607 getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE), 608 frozenset(extensions), 609 cache_size=0, 610 auto_reload=False 611 ) 612 613 if getbool(options, 'newstyle_gettext'): 614 environment.newstyle_gettext = True 615 616 source = fileobj.read().decode(options.get('encoding', 'utf-8')) 617 try: 618 node = environment.parse(source) 619 tokens = list(environment.lex(environment.preprocess(source))) 620 except TemplateSyntaxError as e: 621 if not silent: 622 raise 623 # skip templates with syntax errors 624 return 625 626 finder = _CommentFinder(tokens, comment_tags) 627 for lineno, func, message in extract_from_ast(node, keywords): 628 yield lineno, func, message, finder.find_comments(lineno) 629 630 631 #: nicer import names 632 i18n = InternationalizationExtension 633 do = ExprStmtExtension 634 loopcontrols = LoopControlExtension 635 with_ = WithExtension 636 autoescape = AutoEscapeExtension 637
