web.web.template

1 """ 2 simple, elegant templating 3 (part of web.py) 4 5 Template design: 6 7 Template string is split into tokens and the tokens are combined into nodes. 8 Parse tree is a nodelist. TextNode and ExpressionNode are simple nodes and 9 for-loop, if-loop etc are block nodes, which contain multiple child nodes. 10 11 Each node can emit some python string. python string emitted by the 12 root node is validated for safeeval and executed using python in the given environment. 13 14 Enough care is taken to make sure the generated code and the template has line to line match, 15 so that the error messages can point to exact line number in template. (It doesn't work in some cases still.) 16 17 Grammar: 18 19 template -> defwith sections 20 defwith -> '$def with (' arguments ')' | '' 21 sections -> section* 22 section -> block | assignment | line 23 24 assignment -> '$ ' <assignment expression> 25 line -> (text|expr)* 26 text -> <any characters other than $> 27 expr -> '$' pyexpr | '$(' pyexpr ')' | '${' pyexpr '}' 28 pyexpr -> <python expression> 29 """ 30 31 __all__ = [ 32 "Template", 33 "Render", "render", "frender", 34 "ParseError", "SecurityError", 35 "test" 36 ] 37 38 import tokenize 39 import os 40 import glob 41 import re 42 from UserDict import DictMixin 43 44 from utils import storage, safeunicode, safestr, re_compile 45 from webapi import config 46 from net import websafe 47

48 -def splitline(text):

49 r""" 50 Splits the given text at newline. 51 52 >>> splitline('foo\nbar') 53 ('foo\n', 'bar') 54 >>> splitline('foo') 55 ('foo', '') 56 >>> splitline('') 57 ('', '') 58 """ 59 index = text.find('\n') + 1 60 if index: 61 return text[:index], text[index:] 62 else: 63 return text, ''

64

65 -class Parser:

66 """Parser Base. 67 """

68 - def __init__(self):

69 self.statement_nodes = STATEMENT_NODES 70 self.keywords = KEYWORDS

71

72 - def parse(self, text, name="<template>"):

73 self.text = text 74 self.name = name 75 76 defwith, text = self.read_defwith(text) 77 suite = self.read_suite(text) 78 return DefwithNode(defwith, suite)

79

80 - def read_defwith(self, text):

81 if text.startswith('$def with'): 82 defwith, text = splitline(text) 83 defwith = defwith[1:].strip() # strip $ and spaces 84 return defwith, text 85 else: 86 return '', text

87

88 - def read_section(self, text):

89 r"""Reads one section from the given text. 90 91 section -> block | assignment | line 92 93 >>> read_section = Parser().read_section 94 >>> read_section('foo\nbar\n') 95 (<line: [t'foo\n']>, 'bar\n') 96 >>> read_section('$ a = b + 1\nfoo\n') 97 (<assignment: 'a = b + 1'>, 'foo\n') 98 99 read_section('$for in range(10):\n hello $i\nfoo) 100 """ 101 if text.lstrip(' ').startswith('$'): 102 index = text.index('$') 103 begin_indent, text2 = text[:index], text[index+1:] 104 ahead = self.python_lookahead(text2) 105 106 if ahead == 'var': 107 return self.read_var(text2) 108 elif ahead in self.statement_nodes: 109 return self.read_block_section(text2, begin_indent) 110 elif ahead in self.keywords: 111 return self.read_keyword(text2) 112 elif ahead.strip() == '': 113 # assignments starts with a space after $ 114 # ex: $ a = b + 2 115 return self.read_assignment(text2) 116 return self.readline(text)

117

118 - def read_var(self, text):

119 r"""Reads a var statement. 120 121 >>> read_var = Parser().read_var 122 >>> read_var('var x=10\nfoo') 123 (<var: x = 10>, 'foo') 124 >>> read_var('var x: hello $name\nfoo') 125 (<var: x = join_(u'hello ', escape_(name, True))>, 'foo') 126 """ 127 line, text = splitline(text) 128 tokens = self.python_tokens(line) 129 if len(tokens) < 4: 130 raise SyntaxError('Invalid var statement') 131 132 name = tokens[1] 133 sep = tokens[2] 134 value = line.split(sep, 1)[1].strip() 135 136 if sep == '=': 137 pass # no need to process value 138 elif sep == ':': 139 #@@ Hack for backward-compatability 140 if tokens[3] == '\n': # multi-line var statement 141 block, text = self.read_indented_block(text, ' ') 142 lines = [self.readline(x)[0] for x in block.splitlines()] 143 nodes = [] 144 for x in lines: 145 nodes.extend(x.nodes) 146 nodes.append(TextNode('\n')) 147 else: # single-line var statement 148 linenode, _ = self.readline(value) 149 nodes = linenode.nodes 150 parts = [node.emit('') for node in nodes] 151 value = "join_(%s)" % ", ".join(parts) 152 else: 153 raise SyntaxError('Invalid var statement') 154 return VarNode(name, value), text

155

156 - def read_suite(self, text):

157 r"""Reads section by section till end of text. 158 159 >>> read_suite = Parser().read_suite 160 >>> read_suite('hello $name\nfoo\n') 161 [<line: [t'hello ', $name, t'\n']>, <line: [t'foo\n']>] 162 """ 163 sections = [] 164 while text: 165 section, text = self.read_section(text) 166 sections.append(section) 167 return SuiteNode(sections)

168

169 - def readline(self, text):

170 r"""Reads one line from the text. Newline is supressed if the line ends with \. 171 172 >>> readline = Parser().readline 173 >>> readline('hello $name!\nbye!') 174 (<line: [t'hello ', $name, t'!\n']>, 'bye!') 175 >>> readline('hello $name!\\\nbye!') 176 (<line: [t'hello ', $name, t'!']>, 'bye!') 177 >>> readline('$f()\n\n') 178 (<line: [$f(), t'\n']>, '\n') 179 """ 180 line, text = splitline(text) 181 182 # supress new line if line ends with \ 183 if line.endswith('\\\n'): 184 line = line[:-2] 185 186 nodes = [] 187 while line: 188 node, line = self.read_node(line) 189 nodes.append(node) 190 191 return LineNode(nodes), text

192

193 - def read_node(self, text):

194 r"""Reads a node from the given text and returns the node and remaining text. 195 196 >>> read_node = Parser().read_node 197 >>> read_node('hello $name') 198 (t'hello ', '$name') 199 >>> read_node('$name') 200 ($name, '') 201 """ 202 if text.startswith('$$'): 203 return TextNode('$'), text[2:] 204 elif text.startswith('$#'): # comment 205 line, text = splitline(text) 206 return TextNode('\n'), text 207 elif text.startswith('$'): 208 text = text[1:] # strip $ 209 if text.startswith(':'): 210 escape = False 211 text = text[1:] # strip : 212 else: 213 escape = True 214 return self.read_expr(text, escape=escape) 215 else: 216 return self.read_text(text)

217

218 - def read_text(self, text):

219 r"""Reads a text node from the given text. 220 221 >>> read_text = Parser().read_text 222 >>> read_text('hello $name') 223 (t'hello ', '$name') 224 """ 225 index = text.find('$') 226 if index < 0: 227 return TextNode(text), '' 228 else: 229 return TextNode(text[:index]), text[index:]

230

231 - def read_keyword(self, text):

232 line, text = splitline(text) 233 return StatementNode(line.strip() + "\n"), text

234

235 - def read_expr(self, text, escape=True):

236 """Reads a python expression from the text and returns the expression and remaining text. 237 238 expr -> simple_expr | paren_expr 239 simple_expr -> id extended_expr 240 extended_expr -> attr_access | paren_expr extended_expr | '' 241 attr_access -> dot id extended_expr 242 paren_expr -> [ tokens ] | ( tokens ) | { tokens } 243 244 >>> read_expr = Parser().read_expr 245 >>> read_expr("name") 246 ($name, '') 247 >>> read_expr("a.b and c") 248 ($a.b, ' and c') 249 >>> read_expr("a. b") 250 ($a, '. b') 251 >>> read_expr("name</h1>") 252 ($name, '</h1>') 253 >>> read_expr("(limit)ing") 254 ($(limit), 'ing') 255 >>> read_expr('a[1, 2][:3].f(1+2, "weird string[).", 3 + 4) done.') 256 ($a[1, 2][:3].f(1+2, "weird string[).", 3 + 4), ' done.') 257 """ 258 def simple_expr(): 259 identifier() 260 extended_expr()

261 262 def identifier(): 263 tokens.next()

264 265 def extended_expr(): 266 lookahead = tokens.lookahead() 267 if lookahead is None: 268 return 269 elif lookahead.value == '.': 270 attr_access() 271 elif lookahead.value in parens: 272 paren_expr() 273 extended_expr() 274 else: 275 return 276 277 def attr_access(): 278 from token import NAME # python token constants 279 dot = tokens.lookahead() 280 if tokens.lookahead2().type == NAME: 281 tokens.next() # consume dot 282 identifier() 283 extended_expr() 284 285 def paren_expr(): 286 begin = tokens.next().value 287 end = parens[begin] 288 while True: 289 if tokens.lookahead().value in parens: 290 paren_expr() 291 else: 292 t = tokens.next() 293 if t.value == end: 294 break 295 return 296 297 parens = { 298 "(": ")", 299 "[": "]", 300 "{": "}" 301 } 302 303 def get_tokens(text): 304 """tokenize text using python tokenizer. 305 Python tokenizer ignores spaces, but they might be important in some cases. 306 This function introduces dummy space tokens when it identifies any ignored space. 307 Each token is a storage object containing type, value, begin and end. 308 """ 309 readline = iter([text]).next 310 end = None 311 for t in tokenize.generate_tokens(readline): 312 t = storage(type=t[0], value=t[1], begin=t[2], end=t[3]) 313 if end is not None and end != t.begin: 314 _, x1 = end 315 _, x2 = t.begin 316 yield storage(type=-1, value=text[x1:x2], begin=end, end=t.begin) 317 end = t.end 318 yield t 319 320 class BetterIter: 321 """Iterator like object with 2 support for 2 look aheads.""" 322 def __init__(self, items): 323 self.iteritems = iter(items) 324 self.items = [] 325 self.position = 0 326 self.current_item = None 327 328 def lookahead(self): 329 if len(self.items) <= self.position: 330 self.items.append(self._next()) 331 return self.items[self.position] 332 333 def _next(self): 334 try: 335 return self.iteritems.next() 336 except StopIteration: 337 return None 338 339 def lookahead2(self): 340 if len(self.items) <= self.position+1: 341 self.items.append(self._next()) 342 return self.items[self.position+1] 343 344 def next(self): 345 self.current_item = self.lookahead() 346 self.position += 1 347 return self.current_item 348 349 tokens = BetterIter(get_tokens(text)) 350 351 if tokens.lookahead().value in parens: 352 paren_expr() 353 else: 354 simple_expr() 355 row, col = tokens.current_item.end 356 return ExpressionNode(text[:col], escape=escape), text[col:] 357

358 - def read_assignment(self, text):

359 r"""Reads assignment statement from text. 360 361 >>> read_assignment = Parser().read_assignment 362 >>> read_assignment('a = b + 1\nfoo') 363 (<assignment: 'a = b + 1'>, 'foo') 364 """ 365 line, text = splitline(text) 366 return AssignmentNode(line.strip()), text

367

368 - def python_lookahead(self, text):

369 """Returns the first python token from the given text. 370 371 >>> python_lookahead = Parser().python_lookahead 372 >>> python_lookahead('for i in range(10):') 373 'for' 374 >>> python_lookahead('else:') 375 'else' 376 >>> python_lookahead(' x = 1') 377 ' ' 378 """ 379 readline = iter([text]).next 380 tokens = tokenize.generate_tokens(readline) 381 return tokens.next()[1]

382

383 - def python_tokens(self, text):

384 readline = iter([text]).next 385 tokens = tokenize.generate_tokens(readline) 386 return [t[1] for t in tokens]

387

388 - def read_indented_block(self, text, indent):

389 r"""Read a block of text. A block is what typically follows a for or it statement. 390 It can be in the same line as that of the statement or an indented block. 391 392 >>> read_indented_block = Parser().read_indented_block 393 >>> read_indented_block(' a\n b\nc', ' ') 394 ('a\nb\n', 'c') 395 >>> read_indented_block(' a\n b\n c\nd', ' ') 396 ('a\n b\nc\n', 'd') 397 >>> read_indented_block(' a\n\n b\nc', ' ') 398 ('a\n\n b\n', 'c') 399 """ 400 if indent == '': 401 return '', text 402 403 block = "" 404 while text: 405 line, text2 = splitline(text) 406 if line.strip() == "": 407 block += '\n' 408 elif line.startswith(indent): 409 block += line[len(indent):] 410 else: 411 break 412 text = text2 413 return block, text

414

415 - def read_statement(self, text):

416 r"""Reads a python statement. 417 418 >>> read_statement = Parser().read_statement 419 >>> read_statement('for i in range(10): hello $name') 420 ('for i in range(10):', ' hello $name') 421 """ 422 tok = PythonTokenizer(text) 423 tok.consume_till(':') 424 return text[:tok.index], text[tok.index:]

425

426 - def read_block_section(self, text, begin_indent=''):

427 r""" 428 >>> read_block_section = Parser().read_block_section 429 >>> read_block_section('for i in range(10): hello $i\nfoo') 430 (<block: 'for i in range(10):', [<line: [t'hello ', $i, t'\n']>]>, 'foo') 431 >>> read_block_section('for i in range(10):\n hello $i\n foo', begin_indent=' ') 432 (<block: 'for i in range(10):', [<line: [t'hello ', $i, t'\n']>]>, ' foo') 433 >>> read_block_section('for i in range(10):\n hello $i\nfoo') 434 (<block: 'for i in range(10):', [<line: [t'hello ', $i, t'\n']>]>, 'foo') 435 """ 436 line, text = splitline(text) 437 stmt, line = self.read_statement(line) 438 keyword = self.python_lookahead(stmt) 439 440 # if there is some thing left in the line 441 if line.strip(): 442 block = line.lstrip() 443 else: 444 def find_indent(text): 445 rx = re_compile(' +') 446 match = rx.match(text) 447 first_indent = match and match.group(0) 448 return first_indent or ""

449 450 # find the indentation of the block by looking at the first line 451 first_indent = find_indent(text)[len(begin_indent):] 452 453 #TODO: fix this special case 454 if keyword == "code": 455 indent = begin_indent + first_indent 456 else: 457 indent = begin_indent + min(first_indent, INDENT) 458 459 block, text = self.read_indented_block(text, indent) 460 461 return self.create_block_node(keyword, stmt, block, begin_indent), text 462

463 - def create_block_node(self, keyword, stmt, block, begin_indent):

464 if keyword in self.statement_nodes: 465 return self.statement_nodes[keyword](stmt, block, begin_indent) 466 else: 467 raise ParseError, 'Unknown statement: %s' % repr(keyword)

468

469 -class PythonTokenizer:

470 """Utility wrapper over python tokenizer."""

471 - def __init__(self, text):

472 self.text = text 473 readline = iter([text]).next 474 self.tokens = tokenize.generate_tokens(readline) 475 self.index = 0

476

477 - def consume_till(self, delim):

478 """Consumes tokens till colon. 479 480 >>> tok = PythonTokenizer('for i in range(10): hello $i') 481 >>> tok.consume_till(':') 482 >>> tok.text[:tok.index] 483 'for i in range(10):' 484 >>> tok.text[tok.index:] 485 ' hello $i' 486 """ 487 try: 488 while True: 489 t = self.next() 490 if t.value == delim: 491 break 492 elif t.value == '(': 493 self.consume_till(')') 494 elif t.value == '[': 495 self.consume_till(']') 496 elif t.value == '{': 497 self.consume_till('}') 498 499 # if end of line is found, it is an exception. 500 # Since there is no easy way to report the line number, 501 # leave the error reporting to the python parser later 502 #@@ This should be fixed. 503 if t.value == '\n': 504 break 505 except: 506 #raise ParseError, "Expected %s, found end of line." % repr(delim) 507 508 # raising ParseError doesn't show the line number. 509 # if this error is ignored, then it will be caught when compiling the python code. 510 return

511

512 - def next(self):

513 type, t, begin, end, line = self.tokens.next() 514 row, col = end 515 self.index = col 516 return storage(type=type, value=t, begin=begin, end=end)

517

518 -class DefwithNode:

519 - def __init__(self, defwith, suite):

520 if defwith: 521 self.defwith = defwith.replace('with', '__template__') + ':' 522 # offset 3 lines. for __lineoffset__, loop and self. 523 self.defwith += "\n __lineoffset__ = -3" 524 else: 525 self.defwith = 'def __template__():' 526 # offset 4 lines for __template__, __lineoffset__, loop and self. 527 self.defwith += "\n __lineoffset__ = -4" 528 529 self.defwith += "\n loop = ForLoop()" 530 self.defwith += "\n self = TemplateResult(); extend_ = self.extend" 531 self.suite = suite 532 self.end = "\n return self"

533

534 - def emit(self, indent):

535 return self.defwith + self.suite.emit(indent + INDENT) + self.end

536

537 - def __repr__(self):

538 return "<defwith: %s, %s>" % (self.defwith, self.suite)

539

540 -class TextNode:

541 - def __init__(self, value):

542 self.value = value

543

544 - def emit(self, indent):

545 return repr(safeunicode(self.value))

546

547 - def __repr__(self):

548 return 't' + repr(self.value)

549

550 -class ExpressionNode:

551 - def __init__(self, value, escape=True):

552 self.value = value.strip() 553 554 # convert ${...} to $(...) 555 if value.startswith('{') and value.endswith('}'): 556 self.value = '(' + self.value[1:-1] + ')' 557 558 self.escape = escape

559

560 - def emit(self, indent):

561 return 'escape_(%s, %s)' % (self.value, bool(self.escape))

562

563 - def __repr__(self):

564 if self.escape: 565 escape = '' 566 else: 567 escape = ':' 568 return "$%s%s" % (escape, self.value)

569

570 -class AssignmentNode:

571 - def __init__(self, code):

572 self.code = code

573

574 - def emit(self, indent, begin_indent=''):

575 return indent + self.code + "\n"

576

577 - def __repr__(self):

578 return "<assignment: %s>" % repr(self.code)

579

580 -class LineNode:

581 - def __init__(self, nodes):

582 self.nodes = nodes

583

584 - def emit(self, indent, text_indent='', name=''):

585 text = [node.emit('') for node in self.nodes] 586 if text_indent: 587 text = [repr(text_indent)] + text 588 589 return indent + "extend_([%s])\n" % ", ".join(text)

590

591 - def __repr__(self):

592 return "<line: %s>" % repr(self.nodes)

593 594 INDENT = u' ' # 4 spaces 595

596 -class BlockNode:

597 - def __init__(self, stmt, block, begin_indent=''):

598 self.stmt = stmt 599 self.suite = Parser().read_suite(block) 600 self.begin_indent = begin_indent

601

602 - def emit(self, indent, text_indent=''):

603 text_indent = self.begin_indent + text_indent 604 out = indent + self.stmt + self.suite.emit(indent + INDENT, text_indent) 605 return out

606

607 - def __repr__(self):

608 return "<block: %s, %s>" % (repr(self.stmt), repr(self.suite))

609

610 -class ForNode(BlockNode):

611 - def __init__(self, stmt, block, begin_indent=''):

612 self.original_stmt = stmt 613 tok = PythonTokenizer(stmt) 614 tok.consume_till('in') 615 a = stmt[:tok.index] # for i in 616 b = stmt[tok.index:-1] # rest of for stmt excluding : 617 stmt = a + ' loop.setup(' + b.strip() + '):' 618 BlockNode.__init__(self, stmt, block, begin_indent)

619

620 - def __repr__(self):

621 return "<block: %s, %s>" % (repr(self.original_stmt), repr(self.suite))

622

623 -class CodeNode:

624 - def __init__(self, stmt, block, begin_indent=''):

625 # compensate one line for $code: 626 self.code = "\n" + block

627

628 - def emit(self, indent, text_indent=''):

629 import re 630 rx = re.compile('^', re.M) 631 return rx.sub(indent, self.code).rstrip(' ')

632

633 - def __repr__(self):

634 return "<code: %s>" % repr(self.code)

635

636 -class StatementNode:

637 - def __init__(self, stmt):

638 self.stmt = stmt

639

640 - def emit(self, indent):

641 return indent + self.stmt

642

643 - def __repr__(self):

644 return "<stmt: %s>" % repr(self.stmt)

645

646 -class IfNode(BlockNode):

647 pass

648

649 -class ElseNode(BlockNode):

650 pass

651

652 -class ElifNode(BlockNode):

653 pass

654

655 -class DefNode(BlockNode):

656 - def __init__(self, *a, **kw):

657 BlockNode.__init__(self, *a, **kw) 658 659 code = CodeNode("", "") 660 code.code = "self = TemplateResult(); extend_ = self.extend\n" 661 self.suite.sections.insert(0, code) 662 663 code = CodeNode("", "") 664 code.code = "return self\n" 665 self.suite.sections.append(code)

666

667 - def emit(self, indent, text_indent=''):

668 text_indent = self.begin_indent + text_indent 669 out = indent + self.stmt + self.suite.emit(indent + INDENT, text_indent) 670 return indent + "__lineoffset__ -= 3\n" + out

671

672 -class VarNode:

673 - def __init__(self, name, value):

674 self.name = name 675 self.value = value

676

677 - def emit(self, indent, text_indent):

678 return indent + "self[%s] = %s\n" % (repr(self.name), self.value)

679

680 - def __repr__(self):

681 return "<var: %s = %s>" % (self.name, self.value)

682

683 -class SuiteNode:

684 """Suite is a list of sections."""

685 - def __init__(self, sections):

686 self.sections = sections

687

688 - def emit(self, indent, text_indent=''):

689 return "\n" + "".join([s.emit(indent, text_indent) for s in self.sections])

690

691 - def __repr__(self):

692 return repr(self.sections)

693 694 STATEMENT_NODES = { 695 'for': ForNode, 696 'while': BlockNode, 697 'if': IfNode, 698 'elif': ElifNode, 699 'else': ElseNode, 700 'def': DefNode, 701 'code': CodeNode 702 } 703 704 KEYWORDS = [ 705 "pass", 706 "break", 707 "continue", 708 "return" 709 ] 710 711 TEMPLATE_BUILTIN_NAMES = [ 712 "dict", "enumerate", "float", "int", "bool", "list", "long", "reversed", 713 "set", "slice", "tuple", "xrange", 714 "abs", "all", "any", "callable", "chr", "cmp", "divmod", "filter", "hex", 715 "id", "isinstance", "iter", "len", "max", "min", "oct", "ord", "pow", "range", 716 "True", "False", 717 "None", 718 "__import__", # some c-libraries like datetime requires __import__ to present in the namespace 719 ] 720 721 import __builtin__ 722 TEMPLATE_BUILTINS = dict([(name, getattr(__builtin__, name)) for name in TEMPLATE_BUILTIN_NAMES if name in __builtin__.__dict__]) 723

724 -class ForLoop:

725 """ 726 Wrapper for expression in for stament to support loop.xxx helpers. 727 728 >>> loop = ForLoop() 729 >>> for x in loop.setup(['a', 'b', 'c']): 730 ... print loop.index, loop.revindex, loop.parity, x 731 ... 732 1 3 odd a 733 2 2 even b 734 3 1 odd c 735 >>> loop.index 736 Traceback (most recent call last): 737 ... 738 AttributeError: index 739 """

740 - def __init__(self):

741 self._ctx = None

742

743 - def __getattr__(self, name):

744 if self._ctx is None: 745 raise AttributeError, name 746 else: 747 return getattr(self._ctx, name)

748

749 - def setup(self, seq):

750 self._push() 751 return self._ctx.setup(seq)

752

753 - def _push(self):

754 self._ctx = ForLoopContext(self, self._ctx)

755

756 - def _pop(self):

757 self._ctx = self._ctx.parent

758

759 -class ForLoopContext:

760 """Stackable context for ForLoop to support nested for loops. 761 """

762 - def __init__(self, forloop, parent):

763 self._forloop = forloop 764 self.parent = parent

765

766 - def setup(self, seq):

767 try: 768 self.length = len(seq) 769 except: 770 self.length = 0 771 772 self.index = 0 773 for a in seq: 774 self.index += 1 775 yield a 776 self._forloop._pop()

777 778 index0 = property(lambda self: self.index-1) 779 first = property(lambda self: self.index == 1) 780 last = property(lambda self: self.index == self.length) 781 odd = property(lambda self: self.index % 2 == 1) 782 even = property(lambda self: self.index % 2 == 0) 783 parity = property(lambda self: ['odd', 'even'][self.even]) 784 revindex0 = property(lambda self: self.length - self.index) 785 revindex = property(lambda self: self.length - self.index + 1)

786

787 -class BaseTemplate:

788 - def __init__(self, code, filename, filter, globals, builtins):

789 self.filename = filename 790 self.filter = filter 791 self._globals = globals 792 self._builtins = builtins 793 if code: 794 self.t = self._compile(code) 795 else: 796 self.t = lambda: ''

797

798 - def _compile(self, code):

799 env = self.make_env(self._globals or {}, self._builtins) 800 exec(code, env) 801 return env['__template__']

802

803 - def __call__(self, *a, **kw):

804 __hidetraceback__ = True 805 return self.t(*a, **kw)

806

807 - def make_env(self, globals, builtins):

808 return dict(globals, 809 __builtins__=builtins, 810 ForLoop=ForLoop, 811 TemplateResult=TemplateResult, 812 escape_=self._escape, 813 join_=self._join 814 )

815 - def _join(self, *items):

816 return u"".join(items)

817

818 - def _escape(self, value, escape=False):

819 if value is None: 820 value = '' 821 822 value = safeunicode(value) 823 if escape and self.filter: 824 value = self.filter(value) 825 return value

826 827 _htmlquote_re = re.compile(r'[&<>"\']') 828 _htmlquote_d = { 829 u"&": u"&", 830 u"<": u"<", 831 u">": u">", 832 u"'": u"'", 833 u'"': u""", 834 } 835

836 -def websafe(text):

837 r""" 838 Encodes `text` for raw use in HTML. 839 840 >>> websafe(u"<'&\">") 841 u'<'&">' 842 843 Unlike the websafe function in utils.py, this works with unicode text. 844 """ 845 return _htmlquote_re.sub(lambda m: _htmlquote_d[m.group(0)], text)

846 847

848 -class Template(BaseTemplate):

849 CONTENT_TYPES = { 850 '.html' : 'text/html; charset=utf-8', 851 '.xhtml' : 'application/xhtml+xml; charset=utf-8', 852 '.txt' : 'text/plain', 853 } 854 FILTERS = { 855 '.html': websafe, 856 '.xhtml': websafe, 857 '.xml': websafe 858 } 859 globals = {} 860

861 - def __init__(self, text, filename='<template>', filter=None, globals=None, builtins=None, extensions=None):

862 self.extensions = extensions or [] 863 text = Template.normalize_text(text) 864 code = self.compile_template(text, filename) 865 866 _, ext = os.path.splitext(filename) 867 filter = filter or self.FILTERS.get(ext, None) 868 self.content_type = self.CONTENT_TYPES.get(ext, None) 869 870 if globals is None: 871 globals = self.globals 872 if builtins is None: 873 builtins = TEMPLATE_BUILTINS 874 875 BaseTemplate.__init__(self, code=code, filename=filename, filter=filter, globals=globals, builtins=builtins)

876

877 - def normalize_text(text):

878 """Normalizes template text by correcting \r\n, tabs and BOM chars.""" 879 text = text.replace('\r\n', '\n').replace('\r', '\n').expandtabs() 880 if not text.endswith('\n'): 881 text += '\n' 882 883 # ignore BOM chars at the begining of template 884 BOM = '\xef\xbb\xbf' 885 if isinstance(text, str) and text.startswith(BOM): 886 text = text[len(BOM):] 887 888 # support fort \$ for backward-compatibility 889 text = text.replace(r'\$', '$$') 890 return text

891 normalize_text = staticmethod(normalize_text) 892

893 - def __call__(self, *a, **kw):

894 __hidetraceback__ = True 895 import webapi as web 896 if 'headers' in web.ctx and self.content_type: 897 web.header('Content-Type', self.content_type, unique=True) 898 899 return BaseTemplate.__call__(self, *a, **kw)

900

901 - def generate_code(text, filename, parser=None):

902 # parse the text 903 parser = parser or Parser() 904 rootnode = parser.parse(text, filename) 905 906 # generate python code from the parse tree 907 code = rootnode.emit(indent="").strip() 908 return safestr(code)

909 910 generate_code = staticmethod(generate_code) 911

912 - def create_parser(self):

913 p = Parser() 914 for ext in self.extensions: 915 p = ext(p) 916 return p

917

918 - def compile_template(self, template_string, filename):

919 code = Template.generate_code(template_string, filename, parser=self.create_parser()) 920 921 def get_source_line(filename, lineno): 922 try: 923 lines = open(filename).read().splitlines() 924 return lines[lineno] 925 except: 926 return None

927 928 try: 929 # compile the code first to report the errors, if any, with the filename 930 compiled_code = compile(code, filename, 'exec') 931 except SyntaxError, e: 932 # display template line that caused the error along with the traceback. 933 try: 934 e.msg += '\n\nTemplate traceback:\n File %s, line %s\n %s' % \ 935 (repr(e.filename), e.lineno, get_source_line(e.filename, e.lineno-1)) 936 except: 937 pass 938 raise 939 940 # make sure code is safe 941 import compiler 942 ast = compiler.parse(code) 943 SafeVisitor().walk(ast, filename) 944 945 return compiled_code

946

947 -class CompiledTemplate(Template):

948 - def __init__(self, f, filename):

949 Template.__init__(self, '', filename) 950 self.t = f

951

952 - def compile_template(self, *a):

953 return None

954

955 - def _compile(self, *a):

956 return None

957

958 -class Render:

959 """The most preferred way of using templates. 960 961 render = web.template.render('templates') 962 print render.foo() 963 964 Optional parameter can be `base` can be used to pass output of 965 every template through the base template. 966 967 render = web.template.render('templates', base='layout') 968 """

969 - def __init__(self, loc='templates', cache=None, base=None, **keywords):

970 self._loc = loc 971 self._keywords = keywords 972 973 if cache is None: 974 cache = not config.get('debug', False) 975 976 if cache: 977 self._cache = {} 978 else: 979 self._cache = None 980 981 if base and not hasattr(base, '__call__'): 982 # make base a function, so that it can be passed to sub-renders 983 self._base = lambda page: self._template(base)(page) 984 else: 985 self._base = base

986

987 - def _add_global(self, obj, name=None):

988 """Add a global to this rendering instance.""" 989 if 'globals' not in self._keywords: self._keywords['globals'] = {} 990 if not name: 991 name = obj.__name__ 992 self._keywords['globals'][name] = obj

993

994 - def _lookup(self, name):

995 path = os.path.join(self._loc, name) 996 if os.path.isdir(path): 997 return 'dir', path 998 else: 999 path = self._findfile(path) 1000 if path: 1001 return 'file', path 1002 else: 1003 return 'none', None

1004

1005 - def _load_template(self, name):

1006 kind, path = self._lookup(name) 1007 1008 if kind == 'dir': 1009 return Render(path, cache=self._cache is not None, base=self._base, **self._keywords) 1010 elif kind == 'file': 1011 return Template(open(path).read(), filename=path, **self._keywords) 1012 else: 1013 raise AttributeError, "No template named " + name

1014

1015 - def _findfile(self, path_prefix):

1016 p = [f for f in glob.glob(path_prefix + '.*') if not f.endswith('~')] # skip backup files 1017 p.sort() # sort the matches for deterministic order 1018 return p and p[0]

1019

1020 - def _template(self, name):

1021 if self._cache is not None: 1022 if name not in self._cache: 1023 self._cache[name] = self._load_template(name) 1024 return self._cache[name] 1025 else: 1026 return self._load_template(name)

1027

1028 - def __getattr__(self, name):

1029 t = self._template(name) 1030 if self._base and isinstance(t, Template): 1031 def template(*a, **kw): 1032 return self._base(t(*a, **kw))

1033 return template 1034 else: 1035 return self._template(name) 1036

1037 -class GAE_Render(Render):

1038 # Render gets over-written. make a copy here. 1039 super = Render

1040 - def __init__(self, loc, *a, **kw):

1041 GAE_Render.super.__init__(self, loc, *a, **kw) 1042 1043 import types 1044 if isinstance(loc, types.ModuleType): 1045 self.mod = loc 1046 else: 1047 name = loc.rstrip('/').replace('/', '.') 1048 self.mod = __import__(name, None, None, ['x']) 1049 1050 self.mod.__dict__.update(kw.get('builtins', TEMPLATE_BUILTINS)) 1051 self.mod.__dict__.update(Template.globals) 1052 self.mod.__dict__.update(kw.get('globals', {}))

1053

1054 - def _load_template(self, name):

1055 t = getattr(self.mod, name) 1056 import types 1057 if isinstance(t, types.ModuleType): 1058 return GAE_Render(t, cache=self._cache is not None, base=self._base, **self._keywords) 1059 else: 1060 return t

1061 1062 render = Render 1063 # setup render for Google App Engine. 1064 try: 1065 from google import appengine 1066 render = Render = GAE_Render 1067 except ImportError: 1068 pass 1069

1070 -def frender(path, **keywords):

1071 """Creates a template from the given file path. 1072 """ 1073 return Template(open(path).read(), filename=path, **keywords)

1074

1075 -def compile_templates(root):

1076 """Compiles templates to python code.""" 1077 re_start = re_compile('^', re.M) 1078 1079 for dirpath, dirnames, filenames in os.walk(root): 1080 filenames = [f for f in filenames if not f.startswith('.') and not f.endswith('~') and not f.startswith('__init__.py')] 1081 1082 for d in dirnames[:]: 1083 if d.startswith('.'): 1084 dirnames.remove(d) # don't visit this dir 1085 1086 out = open(os.path.join(dirpath, '__init__.py'), 'w') 1087 out.write('from web.template import CompiledTemplate, ForLoop, TemplateResult\n\n') 1088 if dirnames: 1089 out.write("import " + ", ".join(dirnames)) 1090 1091 out.write("_dummy = CompiledTemplate(lambda: None, 'dummy')\n") 1092 out.write("join_ = _dummy._join\n") 1093 out.write("escape_ = _dummy._escape\n") 1094 out.write("\n") 1095 1096 for f in filenames: 1097 path = os.path.join(dirpath, f) 1098 1099 if '.' in f: 1100 name, _ = f.split('.', 1) 1101 else: 1102 name = f 1103 1104 text = open(path).read() 1105 text = Template.normalize_text(text) 1106 code = Template.generate_code(text, path) 1107 1108 code = code.replace("__template__", name, 1) 1109 1110 out.write(code) 1111 1112 out.write('\n\n') 1113 out.write('%s = CompiledTemplate(%s, %s)\n\n' % (name, name, repr(path))) 1114 1115 # create template to make sure it compiles 1116 t = Template(open(path).read(), path) 1117 out.close()

1118

1119 -class ParseError(Exception):

1120 pass

1121

1122 -class SecurityError(Exception):

1123 """The template seems to be trying to do something naughty.""" 1124 pass

1125 1126 # Enumerate all the allowed AST nodes 1127 ALLOWED_AST_NODES = [ 1128 "Add", "And", 1129 # "AssAttr", 1130 "AssList", "AssName", "AssTuple", 1131 # "Assert", 1132 "Assign", "AugAssign", 1133 # "Backquote", 1134 "Bitand", "Bitor", "Bitxor", "Break", 1135 "CallFunc","Class", "Compare", "Const", "Continue", 1136 "Decorators", "Dict", "Discard", "Div", 1137 "Ellipsis", "EmptyNode", 1138 # "Exec", 1139 "Expression", "FloorDiv", "For", 1140 # "From", 1141 "Function", 1142 "GenExpr", "GenExprFor", "GenExprIf", "GenExprInner", 1143 "Getattr", 1144 # "Global", 1145 "If", "IfExp", 1146 # "Import", 1147 "Invert", "Keyword", "Lambda", "LeftShift", 1148 "List", "ListComp", "ListCompFor", "ListCompIf", "Mod", 1149 "Module", 1150 "Mul", "Name", "Not", "Or", "Pass", "Power", 1151 # "Print", "Printnl", "Raise", 1152 "Return", "RightShift", "Slice", "Sliceobj", 1153 "Stmt", "Sub", "Subscript", 1154 # "TryExcept", "TryFinally", 1155 "Tuple", "UnaryAdd", "UnarySub", 1156 "While", "With", "Yield", 1157 ] 1158

1159 -class SafeVisitor(object):

1160 """ 1161 Make sure code is safe by walking through the AST. 1162 1163 Code considered unsafe if: 1164 * it has restricted AST nodes 1165 * it is trying to access resricted attributes 1166 1167 Adopted from http://www.zafar.se/bkz/uploads/safe.txt (public domain, Babar K. Zafar) 1168 """

1169 - def __init__(self):

1170 "Initialize visitor by generating callbacks for all AST node types." 1171 self.errors = []

1172

1173 - def walk(self, ast, filename):

1174 "Validate each node in AST and raise SecurityError if the code is not safe." 1175 self.filename = filename 1176 self.visit(ast) 1177 1178 if self.errors: 1179 raise SecurityError, '\n'.join([str(err) for err in self.errors])

1180

1181 - def visit(self, node, *args):

1182 "Recursively validate node and all of its children." 1183 def classname(obj): 1184 return obj.__class__.__name__

1185 nodename = classname(node) 1186 fn = getattr(self, 'visit' + nodename, None) 1187 1188 if fn: 1189 fn(node, *args) 1190 else: 1191 if nodename not in ALLOWED_AST_NODES: 1192 self.fail(node, *args) 1193 1194 for child in node.getChildNodes(): 1195 self.visit(child, *args)

1196

1197 - def visitName(self, node, *args):

1198 "Disallow any attempts to access a restricted attr." 1199 #self.assert_attr(node.getChildren()[0], node) 1200 pass

1201

1202 - def visitGetattr(self, node, *args):

1203 "Disallow any attempts to access a restricted attribute." 1204 self.assert_attr(node.attrname, node)

1205

1206 - def assert_attr(self, attrname, node):

1207 if self.is_unallowed_attr(attrname): 1208 lineno = self.get_node_lineno(node) 1209 e = SecurityError("%s:%d - access to attribute '%s' is denied" % (self.filename, lineno, attrname)) 1210 self.errors.append(e)

1211

1212 - def is_unallowed_attr(self, name):

1213 return name.startswith('_') \ 1214 or name.startswith('func_') \ 1215 or name.startswith('im_')

1216

1217 - def get_node_lineno(self, node):

1218 return (node.lineno) and node.lineno or 0

1219

1220 - def fail(self, node, *args):

1221 "Default callback for unallowed AST nodes." 1222 lineno = self.get_node_lineno(node) 1223 nodename = node.__class__.__name__ 1224 e = SecurityError("%s:%d - execution of '%s' statements is denied" % (self.filename, lineno, nodename)) 1225 self.errors.append(e)

1226

1227 -class TemplateResult(storage, DictMixin):

1228 """Dictionary like object for storing template output. 1229 1230 A template can specify key-value pairs in the output using 1231 `var` statements. Each `var` statement adds a new key to the 1232 template output and the main output is stored with key 1233 __body__. 1234 1235 >>> d = TemplateResult(__body__='hello, world', x='foo') 1236 >>> d 1237 <TemplateResult: {'__body__': 'hello, world', 'x': 'foo'}> 1238 >>> print d 1239 hello, world 1240 >>> d = TemplateResult() 1241 >>> d.extend([u'hello', u'world']) 1242 >>> d 1243 <TemplateResult: {'__body__': u'helloworld'}> 1244 """

1245 - def __init__(self, *a, **kw):

1246 storage.__init__(self, *a, **kw) 1247 self.setdefault("__body__", None) 1248 1249 # avoiding self._data because it adds as item instead of attr. 1250 self.__dict__["_data"] = [] 1251 self.__dict__["extend"] = self._data.extend

1252

1253 - def __getitem__(self, name):

1254 if name == "__body__" and storage.__getitem__(self, '__body__') is None: 1255 self["__body__"] = u"".join(self._data) 1256 return storage.__getitem__(self, name)

1257

1258 - def __unicode__(self):

1259 return self["__body__"]

1260

1261 - def __str__(self):

1262 return self["__body__"].encode('utf-8')

1263

1264 - def __repr__(self):

1265 self["__body__"] # initialize __body__ if not already initialized 1266 return "<TemplateResult: %s>" % dict.__repr__(self)

1267

1268 -def test():

1269 r"""Doctest for testing template module. 1270 1271 Define a utility function to run template test. 1272 1273 >>> class TestResult: 1274 ... def __init__(self, t): self.t = t 1275 ... def __getattr__(self, name): return getattr(self.t, name) 1276 ... def __repr__(self): return repr(unicode(self)) 1277 ... 1278 >>> def t(code, **keywords): 1279 ... tmpl = Template(code, **keywords) 1280 ... return lambda *a, **kw: TestResult(tmpl(*a, **kw)) 1281 ... 1282 1283 Simple tests. 1284 1285 >>> t('1')() 1286 u'1\n' 1287 >>> t('$def with ()\n1')() 1288 u'1\n' 1289 >>> t('$def with (a)\n$a')(1) 1290 u'1\n' 1291 >>> t('$def with (a=0)\n$a')(1) 1292 u'1\n' 1293 >>> t('$def with (a=0)\n$a')(a=1) 1294 u'1\n' 1295 1296 Test complicated expressions. 1297 1298 >>> t('$def with (x)\n$x.upper()')('hello') 1299 u'HELLO\n' 1300 >>> t('$(2 * 3 + 4 * 5)')() 1301 u'26\n' 1302 >>> t('${2 * 3 + 4 * 5}')() 1303 u'26\n' 1304 >>> t('$def with (limit)\nkeep $(limit)ing.')('go') 1305 u'keep going.\n' 1306 >>> t('$def with (a)\n$a.b[0]')(storage(b=[1])) 1307 u'1\n' 1308 1309 Test html escaping. 1310 1311 >>> t('$def with (x)\n$x', filename='a.html')('<html>') 1312 u'<html>\n' 1313 >>> t('$def with (x)\n$x', filename='a.txt')('<html>') 1314 u'<html>\n' 1315 1316 Test if, for and while. 1317 1318 >>> t('$if 1: 1')() 1319 u'1\n' 1320 >>> t('$if 1:\n 1')() 1321 u'1\n' 1322 >>> t('$if 1:\n 1\\')() 1323 u'1' 1324 >>> t('$if 0: 0\n$elif 1: 1')() 1325 u'1\n' 1326 >>> t('$if 0: 0\n$elif None: 0\n$else: 1')() 1327 u'1\n' 1328 >>> t('$if 0 < 1 and 1 < 2: 1')() 1329 u'1\n' 1330 >>> t('$for x in [1, 2, 3]: $x')() 1331 u'1\n2\n3\n' 1332 >>> t('$def with (d)\n$for k, v in d.iteritems(): $k')({1: 1}) 1333 u'1\n' 1334 >>> t('$for x in [1, 2, 3]:\n\t$x')() 1335 u' 1\n 2\n 3\n' 1336 >>> t('$def with (a)\n$while a and a.pop():1')([1, 2, 3]) 1337 u'1\n1\n1\n' 1338 1339 The space after : must be ignored. 1340 1341 >>> t('$if True: foo')() 1342 u'foo\n' 1343 1344 Test loop.xxx. 1345 1346 >>> t("$for i in range(5):$loop.index, $loop.parity")() 1347 u'1, odd\n2, even\n3, odd\n4, even\n5, odd\n' 1348 >>> t("$for i in range(2):\n $for j in range(2):$loop.parent.parity $loop.parity")() 1349 u'odd odd\nodd even\neven odd\neven even\n' 1350 1351 Test assignment. 1352 1353 >>> t('$ a = 1\n$a')() 1354 u'1\n' 1355 >>> t('$ a = [1]\n$a[0]')() 1356 u'1\n' 1357 >>> t('$ a = {1: 1}\n$a.keys()[0]')() 1358 u'1\n' 1359 >>> t('$ a = []\n$if not a: 1')() 1360 u'1\n' 1361 >>> t('$ a = {}\n$if not a: 1')() 1362 u'1\n' 1363 >>> t('$ a = -1\n$a')() 1364 u'-1\n' 1365 >>> t('$ a = "1"\n$a')() 1366 u'1\n' 1367 1368 Test comments. 1369 1370 >>> t('$# 0')() 1371 u'\n' 1372 >>> t('hello$#comment1\nhello$#comment2')() 1373 u'hello\nhello\n' 1374 >>> t('$#comment0\nhello$#comment1\nhello$#comment2')() 1375 u'\nhello\nhello\n' 1376 1377 Test unicode. 1378 1379 >>> t('$def with (a)\n$a')(u'\u203d') 1380 u'\u203d\n' 1381 >>> t('$def with (a)\n$a')(u'\u203d'.encode('utf-8')) 1382 u'\u203d\n' 1383 >>> t(u'$def with (a)\n$a $:a')(u'\u203d') 1384 u'\u203d \u203d\n' 1385 >>> t(u'$def with ()\nfoo')() 1386 u'foo\n' 1387 >>> def f(x): return x 1388 ... 1389 >>> t(u'$def with (f)\n$:f("x")')(f) 1390 u'x\n' 1391 >>> t('$def with (f)\n$:f("x")')(f) 1392 u'x\n' 1393 1394 Test dollar escaping. 1395 1396 >>> t("Stop, $$money isn't evaluated.")() 1397 u"Stop, $money isn't evaluated.\n" 1398 >>> t("Stop, \$money isn't evaluated.")() 1399 u"Stop, $money isn't evaluated.\n" 1400 1401 Test space sensitivity. 1402 1403 >>> t('$def with (x)\n$x')(1) 1404 u'1\n' 1405 >>> t('$def with(x ,y)\n$x')(1, 1) 1406 u'1\n' 1407 >>> t('$(1 + 2*3 + 4)')() 1408 u'11\n' 1409 1410 Make sure globals are working. 1411 1412 >>> t('$x')() 1413 Traceback (most recent call last): 1414 ... 1415 NameError: global name 'x' is not defined 1416 >>> t('$x', globals={'x': 1})() 1417 u'1\n' 1418 1419 Can't change globals. 1420 1421 >>> t('$ x = 2\n$x', globals={'x': 1})() 1422 u'2\n' 1423 >>> t('$ x = x + 1\n$x', globals={'x': 1})() 1424 Traceback (most recent call last): 1425 ... 1426 UnboundLocalError: local variable 'x' referenced before assignment 1427 1428 Make sure builtins are customizable. 1429 1430 >>> t('$min(1, 2)')() 1431 u'1\n' 1432 >>> t('$min(1, 2)', builtins={})() 1433 Traceback (most recent call last): 1434 ... 1435 NameError: global name 'min' is not defined 1436 1437 Test vars. 1438 1439 >>> x = t('$var x: 1')() 1440 >>> x.x 1441 u'1' 1442 >>> x = t('$var x = 1')() 1443 >>> x.x 1444 1 1445 >>> x = t('$var x: \n foo\n bar')() 1446 >>> x.x 1447 u'foo\nbar\n' 1448 1449 Test BOM chars. 1450 1451 >>> t('\xef\xbb\xbf$def with(x)\n$x')('foo') 1452 u'foo\n' 1453 1454 Test for with weird cases. 1455 1456 >>> t('$for i in range(10)[1:5]:\n $i')() 1457 u'1\n2\n3\n4\n' 1458 >>> t("$for k, v in {'a': 1, 'b': 2}.items():\n $k $v")() 1459 u'a 1\nb 2\n' 1460 >>> t("$for k, v in ({'a': 1, 'b': 2}.items():\n $k $v")() 1461 Traceback (most recent call last): 1462 ... 1463 SyntaxError: invalid syntax 1464 1465 Test datetime. 1466 1467 >>> import datetime 1468 >>> t("$def with (date)\n$date.strftime('%m %Y')")(datetime.datetime(2009, 1, 1)) 1469 u'01 2009\n' 1470 """ 1471 pass

1472 1473 if __name__ == "__main__": 1474 import sys 1475 if '--compile' in sys.argv: 1476 compile_templates(sys.argv[2]) 1477 else: 1478 import doctest 1479 doctest.testmod() 1480

Source Code for Module web.web.template