1 """ 2 A small templating language 3 4 This implements a small templating language for use internally in 5 Paste and Paste Script. This language implements if/elif/else, 6 for/continue/break, expressions, and blocks of Python code. The 7 syntax is:: 8 9 {{any expression (function calls etc)}} 10 {{any expression | filter}} 11 {{for x in y}}...{{endfor}} 12 {{if x}}x{{elif y}}y{{else}}z{{endif}} 13 {{py:x=1}} 14 {{py: 15 def foo(bar): 16 return 'baz' 17 }} 18 {{default var = default_value}} 19 {{# comment}} 20 21 You use this with the ``Template`` class or the ``sub`` shortcut. 22 The ``Template`` class takes the template string and the name of 23 the template (for errors) and a default namespace. Then (like 24 ``string.Template``) you can call the ``tmpl.substitute(**kw)`` 25 method to make a substitution (or ``tmpl.substitute(a_dict)``). 26 27 ``sub(content, **kw)`` substitutes the template immediately. You 28 can use ``__name='tmpl.html'`` to set the name of the template. 29 30 If there are syntax errors ``TemplateError`` will be raised. 31 """ 32 33 import re 34 import six 35 import sys 36 import cgi 37 from six.moves.urllib.parse import quote 38 from paste.util.looper import looper 39 40 __all__ = ['TemplateError', 'Template', 'sub', 'HTMLTemplate', 41 'sub_html', 'html', 'bunch'] 42 43 token_re = re.compile(r'\{\{|\}\}') 44 in_re = re.compile(r'\s+in\s+') 45 var_re = re.compile(r'^[a-z_][a-z0-9_]*$', re.I) 46 47 class TemplateError(Exception): 48 """Exception raised while parsing a template 49 """ 50 51 def __init__(self, message, position, name=None): 52 self.message = message 53 self.position = position 54 self.name = name 55 56 def __str__(self): 57 msg = '%s at line %s column %s' % ( 58 self.message, self.position[0], self.position[1]) 59 if self.name: 60 msg += ' in %s' % self.name 61 return msg 62 63 class _TemplateContinue(Exception): 64 pass 65 66 class _TemplateBreak(Exception): 67 pass 68 69 class Template(object): 70 71 default_namespace = { 72 'start_braces': '{{', 73 'end_braces': '}}', 74 'looper': looper, 75 } 76 77 default_encoding = 'utf8' 78 79 def __init__(self, content, name=None, namespace=None): 80 self.content = content 81 self._unicode = isinstance(content, six.text_type) 82 self.name = name 83 self._parsed = parse(content, name=name) 84 if namespace is None: 85 namespace = {} 86 self.namespace = namespace 87 88 def from_filename(cls, filename, namespace=None, encoding=None): 89 f = open(filename, 'rb') 90 c = f.read() 91 f.close() 92 if encoding: 93 c = c.decode(encoding) 94 return cls(content=c, name=filename, namespace=namespace) 95 96 from_filename = classmethod(from_filename) 97 98 def __repr__(self): 99 return '<%s %s name=%r>' % ( 100 self.__class__.__name__, 101 hex(id(self))[2:], self.name) 102 103 def substitute(self, *args, **kw): 104 if args: 105 if kw: 106 raise TypeError( 107 "You can only give positional *or* keyword arguments") 108 if len(args) > 1: 109 raise TypeError( 110 "You can only give on positional argument") 111 kw = args[0] 112 ns = self.default_namespace.copy() 113 ns.update(self.namespace) 114 ns.update(kw) 115 result = self._interpret(ns) 116 return result 117 118 def _interpret(self, ns): 119 __traceback_hide__ = True 120 parts = [] 121 self._interpret_codes(self._parsed, ns, out=parts) 122 return ''.join(parts) 123 124 def _interpret_codes(self, codes, ns, out): 125 __traceback_hide__ = True 126 for item in codes: 127 if isinstance(item, six.string_types): 128 out.append(item) 129 else: 130 self._interpret_code(item, ns, out) 131 132 def _interpret_code(self, code, ns, out): 133 __traceback_hide__ = True 134 name, pos = code[0], code[1] 135 if name == 'py': 136 self._exec(code[2], ns, pos) 137 elif name == 'continue': 138 raise _TemplateContinue() 139 elif name == 'break': 140 raise _TemplateBreak() 141 elif name == 'for': 142 vars, expr, content = code[2], code[3], code[4] 143 expr = self._eval(expr, ns, pos) 144 self._interpret_for(vars, expr, content, ns, out) 145 elif name == 'cond': 146 parts = code[2:] 147 self._interpret_if(parts, ns, out) 148 elif name == 'expr': 149 parts = code[2].split('|') 150 base = self._eval(parts[0], ns, pos) 151 for part in parts[1:]: 152 func = self._eval(part, ns, pos) 153 base = func(base) 154 out.append(self._repr(base, pos)) 155 elif name == 'default': 156 var, expr = code[2], code[3] 157 if var not in ns: 158 result = self._eval(expr, ns, pos) 159 ns[var] = result 160 elif name == 'comment': 161 return 162 else: 163 assert 0, "Unknown code: %r" % name 164 165 def _interpret_for(self, vars, expr, content, ns, out): 166 __traceback_hide__ = True 167 for item in expr: 168 if len(vars) == 1: 169 ns[vars[0]] = item 170 else: 171 if len(vars) != len(item): 172 raise ValueError( 173 'Need %i items to unpack (got %i items)' 174 % (len(vars), len(item))) 175 for name, value in zip(vars, item): 176 ns[name] = value 177 try: 178 self._interpret_codes(content, ns, out) 179 except _TemplateContinue: 180 continue 181 except _TemplateBreak: 182 break 183 184 def _interpret_if(self, parts, ns, out): 185 __traceback_hide__ = True 186 # @@: if/else/else gets through 187 for part in parts: 188 assert not isinstance(part, six.string_types) 189 name, pos = part[0], part[1] 190 if name == 'else': 191 result = True 192 else: 193 result = self._eval(part[2], ns, pos) 194 if result: 195 self._interpret_codes(part[3], ns, out) 196 break 197 198 def _eval(self, code, ns, pos): 199 __traceback_hide__ = True 200 try: 201 value = eval(code, ns) 202 return value 203 except: 204 exc_info = sys.exc_info() 205 e = exc_info[1] 206 if getattr(e, 'args'): 207 arg0 = e.args[0] 208 else: 209 arg0 = str(e) 210 e.args = (self._add_line_info(arg0, pos),) 211 six.reraise(exc_info[0], e, exc_info[2]) 212 213 def _exec(self, code, ns, pos): 214 __traceback_hide__ = True 215 try: 216 six.exec_(code, ns) 217 except: 218 exc_info = sys.exc_info() 219 e = exc_info[1] 220 e.args = (self._add_line_info(e.args[0], pos),) 221 six.reraise(exc_info[0], e, exc_info[2]) 222 223 def _repr(self, value, pos): 224 __traceback_hide__ = True 225 try: 226 if value is None: 227 return '' 228 if self._unicode: 229 try: 230 value = six.text_type(value) 231 except UnicodeDecodeError: 232 value = str(value) 233 else: 234 value = str(value) 235 except: 236 exc_info = sys.exc_info() 237 e = exc_info[1] 238 e.args = (self._add_line_info(e.args[0], pos),) 239 six.reraise(exc_info[0], e, exc_info[2]) 240 else: 241 if self._unicode and isinstance(value, six.binary_type): 242 if not self.decode_encoding: 243 raise UnicodeDecodeError( 244 'Cannot decode str value %r into unicode ' 245 '(no default_encoding provided)' % value) 246 value = value.decode(self.default_encoding) 247 elif not self._unicode and isinstance(value, six.text_type): 248 if not self.decode_encoding: 249 raise UnicodeEncodeError( 250 'Cannot encode unicode value %r into str ' 251 '(no default_encoding provided)' % value) 252 value = value.encode(self.default_encoding) 253 return value 254 255 256 def _add_line_info(self, msg, pos): 257 msg = "%s at line %s column %s" % ( 258 msg, pos[0], pos[1]) 259 if self.name: 260 msg += " in file %s" % self.name 261 return msg 262 263 def sub(content, **kw): 264 name = kw.get('__name') 265 tmpl = Template(content, name=name) 266 return tmpl.substitute(kw) 267 268 def paste_script_template_renderer(content, vars, filename=None): 269 tmpl = Template(content, name=filename) 270 return tmpl.substitute(vars) 271 272 class bunch(dict): 273 274 def __init__(self, **kw): 275 for name, value in kw.items(): 276 setattr(self, name, value) 277 278 def __setattr__(self, name, value): 279 self[name] = value 280 281 def __getattr__(self, name): 282 try: 283 return self[name] 284 except KeyError: 285 raise AttributeError(name) 286 287 def __getitem__(self, key): 288 if 'default' in self: 289 try: 290 return dict.__getitem__(self, key) 291 except KeyError: 292 return dict.__getitem__(self, 'default') 293 else: 294 return dict.__getitem__(self, key) 295 296 def __repr__(self): 297 items = [ 298 (k, v) for k, v in self.items()] 299 items.sort() 300 return '<%s %s>' % ( 301 self.__class__.__name__, 302 ' '.join(['%s=%r' % (k, v) for k, v in items])) 303 304 ############################################################ 305 ## HTML Templating 306 ############################################################ 307 308 class html(object): 309 def __init__(self, value): 310 self.value = value 311 def __str__(self): 312 return self.value 313 def __repr__(self): 314 return '<%s %r>' % ( 315 self.__class__.__name__, self.value) 316 317 def html_quote(value): 318 if value is None: 319 return '' 320 if not isinstance(value, six.string_types): 321 if hasattr(value, '__unicode__'): 322 value = unicode(value) 323 else: 324 value = str(value) 325 value = cgi.escape(value, 1) 326 if isinstance(value, unicode): 327 value = value.encode('ascii', 'xmlcharrefreplace') 328 return value 329 330 def url(v): 331 if not isinstance(v, six.string_types): 332 if hasattr(v, '__unicode__'): 333 v = unicode(v) 334 else: 335 v = str(v) 336 if isinstance(v, unicode): 337 v = v.encode('utf8') 338 return quote(v) 339 340 def attr(**kw): 341 kw = kw.items() 342 kw.sort() 343 parts = [] 344 for name, value in kw: 345 if value is None: 346 continue 347 if name.endswith('_'): 348 name = name[:-1] 349 parts.append('%s="%s"' % (html_quote(name), html_quote(value))) 350 return html(' '.join(parts)) 351 352 class HTMLTemplate(Template): 353 354 default_namespace = Template.default_namespace.copy() 355 default_namespace.update(dict( 356 html=html, 357 attr=attr, 358 url=url, 359 )) 360 361 def _repr(self, value, pos): 362 plain = Template._repr(self, value, pos) 363 if isinstance(value, html): 364 return plain 365 else: 366 return html_quote(plain) 367 368 def sub_html(content, **kw): 369 name = kw.get('__name') 370 tmpl = HTMLTemplate(content, name=name) 371 return tmpl.substitute(kw) 372 373 374 ############################################################ 375 ## Lexing and Parsing 376 ############################################################ 377 378 def lex(s, name=None, trim_whitespace=True): 379 """ 380 Lex a string into chunks: 381 382 >>> lex('hey') 383 ['hey'] 384 >>> lex('hey {{you}}') 385 ['hey ', ('you', (1, 7))] 386 >>> lex('hey {{') 387 Traceback (most recent call last): 388 ... 389 TemplateError: No }} to finish last expression at line 1 column 7 390 >>> lex('hey }}') 391 Traceback (most recent call last): 392 ... 393 TemplateError: }} outside expression at line 1 column 7 394 >>> lex('hey {{ {{') 395 Traceback (most recent call last): 396 ... 397 TemplateError: {{ inside expression at line 1 column 10 398 399 """ 400 in_expr = False 401 chunks = [] 402 last = 0 403 last_pos = (1, 1) 404 for match in token_re.finditer(s): 405 expr = match.group(0) 406 pos = find_position(s, match.end()) 407 if expr == '{{' and in_expr: 408 raise TemplateError('{{ inside expression', position=pos, 409 name=name) 410 elif expr == '}}' and not in_expr: 411 raise TemplateError('}} outside expression', position=pos, 412 name=name) 413 if expr == '{{': 414 part = s[last:match.start()] 415 if part: 416 chunks.append(part) 417 in_expr = True 418 else: 419 chunks.append((s[last:match.start()], last_pos)) 420 in_expr = False 421 last = match.end() 422 last_pos = pos 423 if in_expr: 424 raise TemplateError('No }} to finish last expression', 425 name=name, position=last_pos) 426 part = s[last:] 427 if part: 428 chunks.append(part) 429 if trim_whitespace: 430 chunks = trim_lex(chunks) 431 return chunks 432 433 statement_re = re.compile(r'^(?:if |elif |else |for |py:)') 434 single_statements = ['endif', 'endfor', 'continue', 'break'] 435 trail_whitespace_re = re.compile(r'\n[\t ]*$') 436 lead_whitespace_re = re.compile(r'^[\t ]*\n') 437 438 def trim_lex(tokens): 439 r""" 440 Takes a lexed set of tokens, and removes whitespace when there is 441 a directive on a line by itself: 442 443 >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False) 444 >>> tokens 445 [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny'] 446 >>> trim_lex(tokens) 447 [('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y'] 448 """ 449 for i in range(len(tokens)): 450 current = tokens[i] 451 if isinstance(tokens[i], six.string_types): 452 # we don't trim this 453 continue 454 item = current[0] 455 if not statement_re.search(item) and item not in single_statements: 456 continue 457 if not i: 458 prev = '' 459 else: 460 prev = tokens[i-1] 461 if i+1 >= len(tokens): 462 next = '' 463 else: 464 next = tokens[i+1] 465 if (not isinstance(next, six.string_types) 466 or not isinstance(prev, six.string_types)): 467 continue 468 if ((not prev or trail_whitespace_re.search(prev)) 469 and (not next or lead_whitespace_re.search(next))): 470 if prev: 471 m = trail_whitespace_re.search(prev) 472 # +1 to leave the leading \n on: 473 prev = prev[:m.start()+1] 474 tokens[i-1] = prev 475 if next: 476 m = lead_whitespace_re.search(next) 477 next = next[m.end():] 478 tokens[i+1] = next 479 return tokens 480 481 482 def find_position(string, index): 483 """Given a string and index, return (line, column)""" 484 leading = string[:index].splitlines() 485 return (len(leading), len(leading[-1])+1) 486 487 def parse(s, name=None): 488 r""" 489 Parses a string into a kind of AST 490 491 >>> parse('{{x}}') 492 [('expr', (1, 3), 'x')] 493 >>> parse('foo') 494 ['foo'] 495 >>> parse('{{if x}}test{{endif}}') 496 [('cond', (1, 3), ('if', (1, 3), 'x', ['test']))] 497 >>> parse('series->{{for x in y}}x={{x}}{{endfor}}') 498 ['series->', ('for', (1, 11), ('x',), 'y', ['x=', ('expr', (1, 27), 'x')])] 499 >>> parse('{{for x, y in z:}}{{continue}}{{endfor}}') 500 [('for', (1, 3), ('x', 'y'), 'z', [('continue', (1, 21))])] 501 >>> parse('{{py:x=1}}') 502 [('py', (1, 3), 'x=1')] 503 >>> parse('{{if x}}a{{elif y}}b{{else}}c{{endif}}') 504 [('cond', (1, 3), ('if', (1, 3), 'x', ['a']), ('elif', (1, 12), 'y', ['b']), ('else', (1, 23), None, ['c']))] 505 506 Some exceptions:: 507 508 >>> parse('{{continue}}') 509 Traceback (most recent call last): 510 ... 511 TemplateError: continue outside of for loop at line 1 column 3 512 >>> parse('{{if x}}foo') 513 Traceback (most recent call last): 514 ... 515 TemplateError: No {{endif}} at line 1 column 3 516 >>> parse('{{else}}') 517 Traceback (most recent call last): 518 ... 519 TemplateError: else outside of an if block at line 1 column 3 520 >>> parse('{{if x}}{{for x in y}}{{endif}}{{endfor}}') 521 Traceback (most recent call last): 522 ... 523 TemplateError: Unexpected endif at line 1 column 25 524 >>> parse('{{if}}{{endif}}') 525 Traceback (most recent call last): 526 ... 527 TemplateError: if with no expression at line 1 column 3 528 >>> parse('{{for x y}}{{endfor}}') 529 Traceback (most recent call last): 530 ... 531 TemplateError: Bad for (no "in") in 'x y' at line 1 column 3 532 >>> parse('{{py:x=1\ny=2}}') 533 Traceback (most recent call last): 534 ... 535 TemplateError: Multi-line py blocks must start with a newline at line 1 column 3 536 """ 537 tokens = lex(s, name=name) 538 result = [] 539 while tokens: 540 next, tokens = parse_expr(tokens, name) 541 result.append(next) 542 return result 543 544 def parse_expr(tokens, name, context=()): 545 if isinstance(tokens[0], six.string_types): 546 return tokens[0], tokens[1:] 547 expr, pos = tokens[0] 548 expr = expr.strip() 549 if expr.startswith('py:'): 550 expr = expr[3:].lstrip(' \t') 551 if expr.startswith('\n'): 552 expr = expr[1:] 553 else: 554 if '\n' in expr: 555 raise TemplateError( 556 'Multi-line py blocks must start with a newline', 557 position=pos, name=name) 558 return ('py', pos, expr), tokens[1:] 559 elif expr in ('continue', 'break'): 560 if 'for' not in context: 561 raise TemplateError( 562 'continue outside of for loop', 563 position=pos, name=name) 564 return (expr, pos), tokens[1:] 565 elif expr.startswith('if '): 566 return parse_cond(tokens, name, context) 567 elif (expr.startswith('elif ') 568 or expr == 'else'): 569 raise TemplateError( 570 '%s outside of an if block' % expr.split()[0], 571 position=pos, name=name) 572 elif expr in ('if', 'elif', 'for'): 573 raise TemplateError( 574 '%s with no expression' % expr, 575 position=pos, name=name) 576 elif expr in ('endif', 'endfor'): 577 raise TemplateError( 578 'Unexpected %s' % expr, 579 position=pos, name=name) 580 elif expr.startswith('for '): 581 return parse_for(tokens, name, context) 582 elif expr.startswith('default '): 583 return parse_default(tokens, name, context) 584 elif expr.startswith('#'): 585 return ('comment', pos, tokens[0][0]), tokens[1:] 586 return ('expr', pos, tokens[0][0]), tokens[1:] 587 588 def parse_cond(tokens, name, context): 589 start = tokens[0][1] 590 pieces = [] 591 context = context + ('if',) 592 while 1: 593 if not tokens: 594 raise TemplateError( 595 'Missing {{endif}}', 596 position=start, name=name) 597 if (isinstance(tokens[0], tuple) 598 and tokens[0][0] == 'endif'): 599 return ('cond', start) + tuple(pieces), tokens[1:] 600 next, tokens = parse_one_cond(tokens, name, context) 601 pieces.append(next) 602 603 def parse_one_cond(tokens, name, context): 604 (first, pos), tokens = tokens[0], tokens[1:] 605 content = [] 606 if first.endswith(':'): 607 first = first[:-1] 608 if first.startswith('if '): 609 part = ('if', pos, first[3:].lstrip(), content) 610 elif first.startswith('elif '): 611 part = ('elif', pos, first[5:].lstrip(), content) 612 elif first == 'else': 613 part = ('else', pos, None, content) 614 else: 615 assert 0, "Unexpected token %r at %s" % (first, pos) 616 while 1: 617 if not tokens: 618 raise TemplateError( 619 'No {{endif}}', 620 position=pos, name=name) 621 if (isinstance(tokens[0], tuple) 622 and (tokens[0][0] == 'endif' 623 or tokens[0][0].startswith('elif ') 624 or tokens[0][0] == 'else')): 625 return part, tokens 626 next, tokens = parse_expr(tokens, name, context) 627 content.append(next) 628 629 def parse_for(tokens, name, context): 630 first, pos = tokens[0] 631 tokens = tokens[1:] 632 context = ('for',) + context 633 content = [] 634 assert first.startswith('for ') 635 if first.endswith(':'): 636 first = first[:-1] 637 first = first[3:].strip() 638 match = in_re.search(first) 639 if not match: 640 raise TemplateError( 641 'Bad for (no "in") in %r' % first, 642 position=pos, name=name) 643 vars = first[:match.start()] 644 if '(' in vars: 645 raise TemplateError( 646 'You cannot have () in the variable section of a for loop (%r)' 647 % vars, position=pos, name=name) 648 vars = tuple([ 649 v.strip() for v in first[:match.start()].split(',') 650 if v.strip()]) 651 expr = first[match.end():] 652 while 1: 653 if not tokens: 654 raise TemplateError( 655 'No {{endfor}}', 656 position=pos, name=name) 657 if (isinstance(tokens[0], tuple) 658 and tokens[0][0] == 'endfor'): 659 return ('for', pos, vars, expr, content), tokens[1:] 660 next, tokens = parse_expr(tokens, name, context) 661 content.append(next) 662 663 def parse_default(tokens, name, context): 664 first, pos = tokens[0] 665 assert first.startswith('default ') 666 first = first.split(None, 1)[1] 667 parts = first.split('=', 1) 668 if len(parts) == 1: 669 raise TemplateError( 670 "Expression must be {{default var=value}}; no = found in %r" % first, 671 position=pos, name=name) 672 var = parts[0].strip() 673 if ',' in var: 674 raise TemplateError( 675 "{{default x, y = ...}} is not supported", 676 position=pos, name=name) 677 if not var_re.search(var): 678 raise TemplateError( 679 "Not a valid variable name for {{default}}: %r" 680 % var, position=pos, name=name) 681 expr = parts[1].strip() 682 return ('default', pos, var, expr), tokens[1:] 683 684 _fill_command_usage = """\ 685 %prog [OPTIONS] TEMPLATE arg=value 686 687 Use py:arg=value to set a Python value; otherwise all values are 688 strings. 689 """ 690 691 def fill_command(args=None): 692 import sys, optparse, pkg_resources, os 693 if args is None: 694 args = sys.argv[1:] 695 dist = pkg_resources.get_distribution('Paste') 696 parser = optparse.OptionParser( 697 version=str(dist), 698 usage=_fill_command_usage) 699 parser.add_option( 700 '-o', '--output', 701 dest='output', 702 metavar="FILENAME", 703 help="File to write output to (default stdout)") 704 parser.add_option( 705 '--html', 706 dest='use_html', 707 action='store_true', 708 help="Use HTML style filling (including automatic HTML quoting)") 709 parser.add_option( 710 '--env', 711 dest='use_env', 712 action='store_true', 713 help="Put the environment in as top-level variables") 714 options, args = parser.parse_args(args) 715 if len(args) < 1: 716 print('You must give a template filename') 717 print(dir(parser)) 718 assert 0 719 template_name = args[0] 720 args = args[1:] 721 vars = {} 722 if options.use_env: 723 vars.update(os.environ) 724 for value in args: 725 if '=' not in value: 726 print('Bad argument: %r' % value) 727 sys.exit(2) 728 name, value = value.split('=', 1) 729 if name.startswith('py:'): 730 name = name[:3] 731 value = eval(value) 732 vars[name] = value 733 if template_name == '-': 734 template_content = sys.stdin.read() 735 template_name = '<stdin>' 736 else: 737 f = open(template_name, 'rb') 738 template_content = f.read() 739 f.close() 740 if options.use_html: 741 TemplateClass = HTMLTemplate 742 else: 743 TemplateClass = Template 744 template = TemplateClass(template_content, name=template_name) 745 result = template.substitute(vars) 746 if options.output: 747 f = open(options.output, 'wb') 748 f.write(result) 749 f.close() 750 else: 751 sys.stdout.write(result) 752 753 if __name__ == '__main__': 754 from paste.util.template import fill_command 755 fill_command() 756 757 758
