1 """A parser for SGML, using the derived class as a static DTD.""" 2 3 # XXX This only supports those SGML features used by HTML. 4 5 # XXX There should be a way to distinguish between PCDATA (parsed 6 # character data -- the normal case), RCDATA (replaceable character 7 # data -- only char and entity references and end tags are special) 8 # and CDATA (character data -- only end tags are special). RCDATA is 9 # not supported at all. 10 11 12 from warnings import warnpy3k 13 warnpy3k("the sgmllib module has been removed in Python 3.0", 14 stacklevel=2) 15 del warnpy3k 16 17 import markupbase 18 import re 19 20 __all__ = ["SGMLParser", "SGMLParseError"] 21 22 # Regular expressions used for parsing 23 24 interesting = re.compile('[&<]') 25 incomplete = re.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|' 26 '<([a-zA-Z][^<>]*|' 27 '/([a-zA-Z][^<>]*)?|' 28 '![^<>]*)?') 29 30 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') 31 charref = re.compile('&#([0-9]+)[^0-9]') 32 33 starttagopen = re.compile('<[>a-zA-Z]') 34 shorttagopen = re.compile('<[a-zA-Z][-.a-zA-Z0-9]*/') 35 shorttag = re.compile('<([a-zA-Z][-.a-zA-Z0-9]*)/([^/]*)/') 36 piclose = re.compile('>') 37 endbracket = re.compile('[<>]') 38 tagfind = re.compile('[a-zA-Z][-_.a-zA-Z0-9]*') 39 attrfind = re.compile( 40 r'\s*([a-zA-Z_][-:.a-zA-Z_0-9]*)(\s*=\s*' 41 r'(\'[^\']*\'|"[^"]*"|[][\-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~\'"@]*))?') 42 43 44 class SGMLParseError(RuntimeError): 45 """Exception raised for all parse errors.""" 46 pass 47 48 49 # SGML parser base class -- find tags and call handler functions. 50 # Usage: p = SGMLParser(); p.feed(data); ...; p.close(). 51 # The dtd is defined by deriving a class which defines methods 52 # with special names to handle tags: start_foo and end_foo to handle 53 # <foo> and </foo>, respectively, or do_foo to handle <foo> by itself. 54 # (Tags are converted to lower case for this purpose.) The data 55 # between tags is passed to the parser by calling self.handle_data() 56 # with some data as argument (the data may be split up in arbitrary 57 # chunks). Entity references are passed by calling 58 # self.handle_entityref() with the entity reference as argument. 59 60 class SGMLParser(markupbase.ParserBase): 61 # Definition of entities -- derived classes may override 62 entity_or_charref = re.compile('&(?:' 63 '([a-zA-Z][-.a-zA-Z0-9]*)|#([0-9]+)' 64 ')(;?)') 65 66 def __init__(self, verbose=0): 67 """Initialize and reset this instance.""" 68 self.verbose = verbose 69 self.reset() 70 71 def reset(self): 72 """Reset this instance. Loses all unprocessed data.""" 73 self.__starttag_text = None 74 self.rawdata = '' 75 self.stack = [] 76 self.lasttag = '???' 77 self.nomoretags = 0 78 self.literal = 0 79 markupbase.ParserBase.reset(self) 80 81 def setnomoretags(self): 82 """Enter literal mode (CDATA) till EOF. 83 84 Intended for derived classes only. 85 """ 86 self.nomoretags = self.literal = 1 87 88 def setliteral(self, *args): 89 """Enter literal mode (CDATA). 90 91 Intended for derived classes only. 92 """ 93 self.literal = 1 94 95 def feed(self, data): 96 """Feed some data to the parser. 97 98 Call this as often as you want, with as little or as much text 99 as you want (may include '\n'). (This just saves the text, 100 all the processing is done by goahead().) 101 """ 102 103 self.rawdata = self.rawdata + data 104 self.goahead(0) 105 106 def close(self): 107 """Handle the remaining data.""" 108 self.goahead(1) 109 110 def error(self, message): 111 raise SGMLParseError(message) 112 113 # Internal -- handle data as far as reasonable. May leave state 114 # and data to be processed by a subsequent call. If 'end' is 115 # true, force handling all data as if followed by EOF marker. 116 def goahead(self, end): 117 rawdata = self.rawdata 118 i = 0 119 n = len(rawdata) 120 while i < n: 121 if self.nomoretags: 122 self.handle_data(rawdata[i:n]) 123 i = n 124 break 125 match = interesting.search(rawdata, i) 126 if match: j = match.start() 127 else: j = n 128 if i < j: 129 self.handle_data(rawdata[i:j]) 130 i = j 131 if i == n: break 132 if rawdata[i] == '<': 133 if starttagopen.match(rawdata, i): 134 if self.literal: 135 self.handle_data(rawdata[i]) 136 i = i+1 137 continue 138 k = self.parse_starttag(i) 139 if k < 0: break 140 i = k 141 continue 142 if rawdata.startswith("</", i): 143 k = self.parse_endtag(i) 144 if k < 0: break 145 i = k 146 self.literal = 0 147 continue 148 if self.literal: 149 if n > (i + 1): 150 self.handle_data("<") 151 i = i+1 152 else: 153 # incomplete 154 break 155 continue 156 if rawdata.startswith("<!--", i): 157 # Strictly speaking, a comment is --.*-- 158 # within a declaration tag <!...>. 159 # This should be removed, 160 # and comments handled only in parse_declaration. 161 k = self.parse_comment(i) 162 if k < 0: break 163 i = k 164 continue 165 if rawdata.startswith("<?", i): 166 k = self.parse_pi(i) 167 if k < 0: break 168 i = i+k 169 continue 170 if rawdata.startswith("<!", i): 171 # This is some sort of declaration; in "HTML as 172 # deployed," this should only be the document type 173 # declaration ("<!DOCTYPE html...>"). 174 k = self.parse_declaration(i) 175 if k < 0: break 176 i = k 177 continue 178 elif rawdata[i] == '&': 179 if self.literal: 180 self.handle_data(rawdata[i]) 181 i = i+1 182 continue 183 match = charref.match(rawdata, i) 184 if match: 185 name = match.group(1) 186 self.handle_charref(name) 187 i = match.end(0) 188 if rawdata[i-1] != ';': i = i-1 189 continue 190 match = entityref.match(rawdata, i) 191 if match: 192 name = match.group(1) 193 self.handle_entityref(name) 194 i = match.end(0) 195 if rawdata[i-1] != ';': i = i-1 196 continue 197 else: 198 self.error('neither < nor & ??') 199 # We get here only if incomplete matches but 200 # nothing else 201 match = incomplete.match(rawdata, i) 202 if not match: 203 self.handle_data(rawdata[i]) 204 i = i+1 205 continue 206 j = match.end(0) 207 if j == n: 208 break # Really incomplete 209 self.handle_data(rawdata[i:j]) 210 i = j 211 # end while 212 if end and i < n: 213 self.handle_data(rawdata[i:n]) 214 i = n 215 self.rawdata = rawdata[i:] 216 # XXX if end: check for empty stack 217 218 # Extensions for the DOCTYPE scanner: 219 _decl_otherchars = '=' 220 221 # Internal -- parse processing instr, return length or -1 if not terminated 222 def parse_pi(self, i): 223 rawdata = self.rawdata 224 if rawdata[i:i+2] != '<?': 225 self.error('unexpected call to parse_pi()') 226 match = piclose.search(rawdata, i+2) 227 if not match: 228 return -1 229 j = match.start(0) 230 self.handle_pi(rawdata[i+2: j]) 231 j = match.end(0) 232 return j-i 233 234 def get_starttag_text(self): 235 return self.__starttag_text 236 237 # Internal -- handle starttag, return length or -1 if not terminated 238 def parse_starttag(self, i): 239 self.__starttag_text = None 240 start_pos = i 241 rawdata = self.rawdata 242 if shorttagopen.match(rawdata, i): 243 # SGML shorthand: <tag/data/ == <tag>data</tag> 244 # XXX Can data contain &... (entity or char refs)? 245 # XXX Can data contain < or > (tag characters)? 246 # XXX Can there be whitespace before the first /? 247 match = shorttag.match(rawdata, i) 248 if not match: 249 return -1 250 tag, data = match.group(1, 2) 251 self.__starttag_text = '<%s/' % tag 252 tag = tag.lower() 253 k = match.end(0) 254 self.finish_shorttag(tag, data) 255 self.__starttag_text = rawdata[start_pos:match.end(1) + 1] 256 return k 257 # XXX The following should skip matching quotes (' or ") 258 # As a shortcut way to exit, this isn't so bad, but shouldn't 259 # be used to locate the actual end of the start tag since the 260 # < or > characters may be embedded in an attribute value. 261 match = endbracket.search(rawdata, i+1) 262 if not match: 263 return -1 264 j = match.start(0) 265 # Now parse the data between i+1 and j into a tag and attrs 266 attrs = [] 267 if rawdata[i:i+2] == '<>': 268 # SGML shorthand: <> == <last open tag seen> 269 k = j 270 tag = self.lasttag 271 else: 272 match = tagfind.match(rawdata, i+1) 273 if not match: 274 self.error('unexpected call to parse_starttag') 275 k = match.end(0) 276 tag = rawdata[i+1:k].lower() 277 self.lasttag = tag 278 while k < j: 279 match = attrfind.match(rawdata, k) 280 if not match: break 281 attrname, rest, attrvalue = match.group(1, 2, 3) 282 if not rest: 283 attrvalue = attrname 284 else: 285 if (attrvalue[:1] == "'" == attrvalue[-1:] or 286 attrvalue[:1] == '"' == attrvalue[-1:]): 287 # strip quotes 288 attrvalue = attrvalue[1:-1] 289 attrvalue = self.entity_or_charref.sub( 290 self._convert_ref, attrvalue) 291 attrs.append((attrname.lower(), attrvalue)) 292 k = match.end(0) 293 if rawdata[j] == '>': 294 j = j+1 295 self.__starttag_text = rawdata[start_pos:j] 296 self.finish_starttag(tag, attrs) 297 return j 298 299 # Internal -- convert entity or character reference 300 def _convert_ref(self, match): 301 if match.group(2): 302 return self.convert_charref(match.group(2)) or \ 303 '&#%s%s' % match.groups()[1:] 304 elif match.group(3): 305 return self.convert_entityref(match.group(1)) or \ 306 '&%s;' % match.group(1) 307 else: 308 return '&%s' % match.group(1) 309 310 # Internal -- parse endtag 311 def parse_endtag(self, i): 312 rawdata = self.rawdata 313 match = endbracket.search(rawdata, i+1) 314 if not match: 315 return -1 316 j = match.start(0) 317 tag = rawdata[i+2:j].strip().lower() 318 if rawdata[j] == '>': 319 j = j+1 320 self.finish_endtag(tag) 321 return j 322 323 # Internal -- finish parsing of <tag/data/ (same as <tag>data</tag>) 324 def finish_shorttag(self, tag, data): 325 self.finish_starttag(tag, []) 326 self.handle_data(data) 327 self.finish_endtag(tag) 328 329 # Internal -- finish processing of start tag 330 # Return -1 for unknown tag, 0 for open-only tag, 1 for balanced tag 331 def finish_starttag(self, tag, attrs): 332 try: 333 method = getattr(self, 'start_' + tag) 334 except AttributeError: 335 try: 336 method = getattr(self, 'do_' + tag) 337 except AttributeError: 338 self.unknown_starttag(tag, attrs) 339 return -1 340 else: 341 self.handle_starttag(tag, method, attrs) 342 return 0 343 else: 344 self.stack.append(tag) 345 self.handle_starttag(tag, method, attrs) 346 return 1 347 348 # Internal -- finish processing of end tag 349 def finish_endtag(self, tag): 350 if not tag: 351 found = len(self.stack) - 1 352 if found < 0: 353 self.unknown_endtag(tag) 354 return 355 else: 356 if tag not in self.stack: 357 try: 358 method = getattr(self, 'end_' + tag) 359 except AttributeError: 360 self.unknown_endtag(tag) 361 else: 362 self.report_unbalanced(tag) 363 return 364 found = len(self.stack) 365 for i in range(found): 366 if self.stack[i] == tag: found = i 367 while len(self.stack) > found: 368 tag = self.stack[-1] 369 try: 370 method = getattr(self, 'end_' + tag) 371 except AttributeError: 372 method = None 373 if method: 374 self.handle_endtag(tag, method) 375 else: 376 self.unknown_endtag(tag) 377 del self.stack[-1] 378 379 # Overridable -- handle start tag 380 def handle_starttag(self, tag, method, attrs): 381 method(attrs) 382 383 # Overridable -- handle end tag 384 def handle_endtag(self, tag, method): 385 method() 386 387 # Example -- report an unbalanced </...> tag. 388 def report_unbalanced(self, tag): 389 if self.verbose: 390 print '*** Unbalanced </' + tag + '>' 391 print '*** Stack:', self.stack 392 393 def convert_charref(self, name): 394 """Convert character reference, may be overridden.""" 395 try: 396 n = int(name) 397 except ValueError: 398 return 399 if not 0 <= n <= 127: 400 return 401 return self.convert_codepoint(n) 402 403 def convert_codepoint(self, codepoint): 404 return chr(codepoint) 405 406 def handle_charref(self, name): 407 """Handle character reference, no need to override.""" 408 replacement = self.convert_charref(name) 409 if replacement is None: 410 self.unknown_charref(name) 411 else: 412 self.handle_data(replacement) 413 414 # Definition of entities -- derived classes may override 415 entitydefs = \ 416 {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''} 417 418 def convert_entityref(self, name): 419 """Convert entity references. 420 421 As an alternative to overriding this method; one can tailor the 422 results by setting up the self.entitydefs mapping appropriately. 423 """ 424 table = self.entitydefs 425 if name in table: 426 return table[name] 427 else: 428 return 429 430 def handle_entityref(self, name): 431 """Handle entity references, no need to override.""" 432 replacement = self.convert_entityref(name) 433 if replacement is None: 434 self.unknown_entityref(name) 435 else: 436 self.handle_data(replacement) 437 438 # Example -- handle data, should be overridden 439 def handle_data(self, data): 440 pass 441 442 # Example -- handle comment, could be overridden 443 def handle_comment(self, data): 444 pass 445 446 # Example -- handle declaration, could be overridden 447 def handle_decl(self, decl): 448 pass 449 450 # Example -- handle processing instruction, could be overridden 451 def handle_pi(self, data): 452 pass 453 454 # To be overridden -- handlers for unknown objects 455 def unknown_starttag(self, tag, attrs): pass 456 def unknown_endtag(self, tag): pass 457 def unknown_charref(self, ref): pass 458 def unknown_entityref(self, ref): pass 459 460 461 class TestSGMLParser(SGMLParser): 462 463 def __init__(self, verbose=0): 464 self.testdata = "" 465 SGMLParser.__init__(self, verbose) 466 467 def handle_data(self, data): 468 self.testdata = self.testdata + data 469 if len(repr(self.testdata)) >= 70: 470 self.flush() 471 472 def flush(self): 473 data = self.testdata 474 if data: 475 self.testdata = "" 476 print 'data:', repr(data) 477 478 def handle_comment(self, data): 479 self.flush() 480 r = repr(data) 481 if len(r) > 68: 482 r = r[:32] + '...' + r[-32:] 483 print 'comment:', r 484 485 def unknown_starttag(self, tag, attrs): 486 self.flush() 487 if not attrs: 488 print 'start tag: <' + tag + '>' 489 else: 490 print 'start tag: <' + tag, 491 for name, value in attrs: 492 print name + '=' + '"' + value + '"', 493 print '>' 494 495 def unknown_endtag(self, tag): 496 self.flush() 497 print 'end tag: </' + tag + '>' 498 499 def unknown_entityref(self, ref): 500 self.flush() 501 print '*** unknown entity ref: &' + ref + ';' 502 503 def unknown_charref(self, ref): 504 self.flush() 505 print '*** unknown char ref: &#' + ref + ';' 506 507 def unknown_decl(self, data): 508 self.flush() 509 print '*** unknown decl: [' + data + ']' 510 511 def close(self): 512 SGMLParser.close(self) 513 self.flush() 514 515 516 def test(args = None): 517 import sys 518 519 if args is None: 520 args = sys.argv[1:] 521 522 if args and args[0] == '-s': 523 args = args[1:] 524 klass = SGMLParser 525 else: 526 klass = TestSGMLParser 527 528 if args: 529 file = args[0] 530 else: 531 file = 'test.html' 532 533 if file == '-': 534 f = sys.stdin 535 else: 536 try: 537 f = open(file, 'r') 538 except IOError, msg: 539 print file, ":", msg 540 sys.exit(1) 541 542 data = f.read() 543 if f is not sys.stdin: 544 f.close() 545 546 x = klass() 547 for c in data: 548 x.feed(c) 549 x.close() 550 551 552 if __name__ == '__main__': 553 test() 554
