1 """A parser for HTML and XHTML.""" 2 3 # This file is based on sgmllib.py, but the API is slightly different. 4 5 # XXX There should be a way to distinguish between PCDATA (parsed 6 # character data -- the normal case), RCDATA (replaceable character 7 # data -- only char and entity references and end tags are special) 8 # and CDATA (character data -- only end tags are special). 9 10 11 import markupbase 12 import re 13 14 # Regular expressions used for parsing 15 16 interesting_normal = re.compile('[&<]') 17 incomplete = re.compile('&[a-zA-Z#]') 18 19 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') 20 charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') 21 22 starttagopen = re.compile('<[a-zA-Z]') 23 piclose = re.compile('>') 24 commentclose = re.compile(r'--\s*>') 25 tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*') 26 # see http://www.w3.org/TR/html5/tokenization.html#tag-open-state 27 # and http://www.w3.org/TR/html5/tokenization.html#tag-name-state 28 tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*') 29 30 attrfind = re.compile( 31 r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' 32 r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') 33 34 locatestarttagend = re.compile(r""" 35 <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name 36 (?:[\s/]* # optional whitespace before attribute name 37 (?:(?<=['"\s/])[^\s/>][^\s/=>]* # attribute name 38 (?:\s*=+\s* # value indicator 39 (?:'[^']*' # LITA-enclosed value 40 |"[^"]*" # LIT-enclosed value 41 |(?!['"])[^>\s]* # bare value 42 ) 43 )?(?:\s|/(?!>))* 44 )* 45 )? 46 \s* # trailing whitespace 47 """, re.VERBOSE) 48 endendtag = re.compile('>') 49 # the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between 50 # </ and the tag name, so maybe this should be fixed 51 endtagfind = re.compile('</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>') 52 53 54 class HTMLParseError(Exception): 55 """Exception raised for all parse errors.""" 56 57 def __init__(self, msg, position=(None, None)): 58 assert msg 59 self.msg = msg 60 self.lineno = position[0] 61 self.offset = position[1] 62 63 def __str__(self): 64 result = self.msg 65 if self.lineno is not None: 66 result = result + ", at line %d" % self.lineno 67 if self.offset is not None: 68 result = result + ", column %d" % (self.offset + 1) 69 return result 70 71 72 class HTMLParser(markupbase.ParserBase): 73 """Find tags and other markup and call handler functions. 74 75 Usage: 76 p = HTMLParser() 77 p.feed(data) 78 ... 79 p.close() 80 81 Start tags are handled by calling self.handle_starttag() or 82 self.handle_startendtag(); end tags by self.handle_endtag(). The 83 data between tags is passed from the parser to the derived class 84 by calling self.handle_data() with the data as argument (the data 85 may be split up in arbitrary chunks). Entity references are 86 passed by calling self.handle_entityref() with the entity 87 reference as the argument. Numeric character references are 88 passed to self.handle_charref() with the string containing the 89 reference as the argument. 90 """ 91 92 CDATA_CONTENT_ELEMENTS = ("script", "style") 93 94 95 def __init__(self): 96 """Initialize and reset this instance.""" 97 self.reset() 98 99 def reset(self): 100 """Reset this instance. Loses all unprocessed data.""" 101 self.rawdata = '' 102 self.lasttag = '???' 103 self.interesting = interesting_normal 104 self.cdata_elem = None 105 markupbase.ParserBase.reset(self) 106 107 def feed(self, data): 108 r"""Feed data to the parser. 109 110 Call this as often as you want, with as little or as much text 111 as you want (may include '\n'). 112 """ 113 self.rawdata = self.rawdata + data 114 self.goahead(0) 115 116 def close(self): 117 """Handle any buffered data.""" 118 self.goahead(1) 119 120 def error(self, message): 121 raise HTMLParseError(message, self.getpos()) 122 123 __starttag_text = None 124 125 def get_starttag_text(self): 126 """Return full source of start tag: '<...>'.""" 127 return self.__starttag_text 128 129 def set_cdata_mode(self, elem): 130 self.cdata_elem = elem.lower() 131 self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I) 132 133 def clear_cdata_mode(self): 134 self.interesting = interesting_normal 135 self.cdata_elem = None 136 137 # Internal -- handle data as far as reasonable. May leave state 138 # and data to be processed by a subsequent call. If 'end' is 139 # true, force handling all data as if followed by EOF marker. 140 def goahead(self, end): 141 rawdata = self.rawdata 142 i = 0 143 n = len(rawdata) 144 while i < n: 145 match = self.interesting.search(rawdata, i) # < or & 146 if match: 147 j = match.start() 148 else: 149 if self.cdata_elem: 150 break 151 j = n 152 if i < j: self.handle_data(rawdata[i:j]) 153 i = self.updatepos(i, j) 154 if i == n: break 155 startswith = rawdata.startswith 156 if startswith('<', i): 157 if starttagopen.match(rawdata, i): # < + letter 158 k = self.parse_starttag(i) 159 elif startswith("</", i): 160 k = self.parse_endtag(i) 161 elif startswith("<!--", i): 162 k = self.parse_comment(i) 163 elif startswith("<?", i): 164 k = self.parse_pi(i) 165 elif startswith("<!", i): 166 k = self.parse_html_declaration(i) 167 elif (i + 1) < n: 168 self.handle_data("<") 169 k = i + 1 170 else: 171 break 172 if k < 0: 173 if not end: 174 break 175 k = rawdata.find('>', i + 1) 176 if k < 0: 177 k = rawdata.find('<', i + 1) 178 if k < 0: 179 k = i + 1 180 else: 181 k += 1 182 self.handle_data(rawdata[i:k]) 183 i = self.updatepos(i, k) 184 elif startswith("&#", i): 185 match = charref.match(rawdata, i) 186 if match: 187 name = match.group()[2:-1] 188 self.handle_charref(name) 189 k = match.end() 190 if not startswith(';', k-1): 191 k = k - 1 192 i = self.updatepos(i, k) 193 continue 194 else: 195 if ";" in rawdata[i:]: #bail by consuming &# 196 self.handle_data(rawdata[0:2]) 197 i = self.updatepos(i, 2) 198 break 199 elif startswith('&', i): 200 match = entityref.match(rawdata, i) 201 if match: 202 name = match.group(1) 203 self.handle_entityref(name) 204 k = match.end() 205 if not startswith(';', k-1): 206 k = k - 1 207 i = self.updatepos(i, k) 208 continue 209 match = incomplete.match(rawdata, i) 210 if match: 211 # match.group() will contain at least 2 chars 212 if end and match.group() == rawdata[i:]: 213 self.error("EOF in middle of entity or char ref") 214 # incomplete 215 break 216 elif (i + 1) < n: 217 # not the end of the buffer, and can't be confused 218 # with some other construct 219 self.handle_data("&") 220 i = self.updatepos(i, i + 1) 221 else: 222 break 223 else: 224 assert 0, "interesting.search() lied" 225 # end while 226 if end and i < n and not self.cdata_elem: 227 self.handle_data(rawdata[i:n]) 228 i = self.updatepos(i, n) 229 self.rawdata = rawdata[i:] 230 231 # Internal -- parse html declarations, return length or -1 if not terminated 232 # See w3.org/TR/html5/tokenization.html#markup-declaration-open-state 233 # See also parse_declaration in _markupbase 234 def parse_html_declaration(self, i): 235 rawdata = self.rawdata 236 if rawdata[i:i+2] != '<!': 237 self.error('unexpected call to parse_html_declaration()') 238 if rawdata[i:i+4] == '<!--': 239 # this case is actually already handled in goahead() 240 return self.parse_comment(i) 241 elif rawdata[i:i+3] == '<![': 242 return self.parse_marked_section(i) 243 elif rawdata[i:i+9].lower() == '<!doctype': 244 # find the closing > 245 gtpos = rawdata.find('>', i+9) 246 if gtpos == -1: 247 return -1 248 self.handle_decl(rawdata[i+2:gtpos]) 249 return gtpos+1 250 else: 251 return self.parse_bogus_comment(i) 252 253 # Internal -- parse bogus comment, return length or -1 if not terminated 254 # see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state 255 def parse_bogus_comment(self, i, report=1): 256 rawdata = self.rawdata 257 if rawdata[i:i+2] not in ('<!', '</'): 258 self.error('unexpected call to parse_comment()') 259 pos = rawdata.find('>', i+2) 260 if pos == -1: 261 return -1 262 if report: 263 self.handle_comment(rawdata[i+2:pos]) 264 return pos + 1 265 266 # Internal -- parse processing instr, return end or -1 if not terminated 267 def parse_pi(self, i): 268 rawdata = self.rawdata 269 assert rawdata[i:i+2] == '<?', 'unexpected call to parse_pi()' 270 match = piclose.search(rawdata, i+2) # > 271 if not match: 272 return -1 273 j = match.start() 274 self.handle_pi(rawdata[i+2: j]) 275 j = match.end() 276 return j 277 278 # Internal -- handle starttag, return end or -1 if not terminated 279 def parse_starttag(self, i): 280 self.__starttag_text = None 281 endpos = self.check_for_whole_start_tag(i) 282 if endpos < 0: 283 return endpos 284 rawdata = self.rawdata 285 self.__starttag_text = rawdata[i:endpos] 286 287 # Now parse the data between i+1 and j into a tag and attrs 288 attrs = [] 289 match = tagfind.match(rawdata, i+1) 290 assert match, 'unexpected call to parse_starttag()' 291 k = match.end() 292 self.lasttag = tag = match.group(1).lower() 293 294 while k < endpos: 295 m = attrfind.match(rawdata, k) 296 if not m: 297 break 298 attrname, rest, attrvalue = m.group(1, 2, 3) 299 if not rest: 300 attrvalue = None 301 elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ 302 attrvalue[:1] == '"' == attrvalue[-1:]: 303 attrvalue = attrvalue[1:-1] 304 if attrvalue: 305 attrvalue = self.unescape(attrvalue) 306 attrs.append((attrname.lower(), attrvalue)) 307 k = m.end() 308 309 end = rawdata[k:endpos].strip() 310 if end not in (">", "/>"): 311 lineno, offset = self.getpos() 312 if "\n" in self.__starttag_text: 313 lineno = lineno + self.__starttag_text.count("\n") 314 offset = len(self.__starttag_text) \ 315 - self.__starttag_text.rfind("\n") 316 else: 317 offset = offset + len(self.__starttag_text) 318 self.handle_data(rawdata[i:endpos]) 319 return endpos 320 if end.endswith('/>'): 321 # XHTML-style empty tag: <span attr="value" /> 322 self.handle_startendtag(tag, attrs) 323 else: 324 self.handle_starttag(tag, attrs) 325 if tag in self.CDATA_CONTENT_ELEMENTS: 326 self.set_cdata_mode(tag) 327 return endpos 328 329 # Internal -- check to see if we have a complete starttag; return end 330 # or -1 if incomplete. 331 def check_for_whole_start_tag(self, i): 332 rawdata = self.rawdata 333 m = locatestarttagend.match(rawdata, i) 334 if m: 335 j = m.end() 336 next = rawdata[j:j+1] 337 if next == ">": 338 return j + 1 339 if next == "/": 340 if rawdata.startswith("/>", j): 341 return j + 2 342 if rawdata.startswith("/", j): 343 # buffer boundary 344 return -1 345 # else bogus input 346 self.updatepos(i, j + 1) 347 self.error("malformed empty start tag") 348 if next == "": 349 # end of input 350 return -1 351 if next in ("abcdefghijklmnopqrstuvwxyz=/" 352 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"): 353 # end of input in or before attribute value, or we have the 354 # '/' from a '/>' ending 355 return -1 356 if j > i: 357 return j 358 else: 359 return i + 1 360 raise AssertionError("we should not get here!") 361 362 # Internal -- parse endtag, return end or -1 if incomplete 363 def parse_endtag(self, i): 364 rawdata = self.rawdata 365 assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag" 366 match = endendtag.search(rawdata, i+1) # > 367 if not match: 368 return -1 369 gtpos = match.end() 370 match = endtagfind.match(rawdata, i) # </ + tag + > 371 if not match: 372 if self.cdata_elem is not None: 373 self.handle_data(rawdata[i:gtpos]) 374 return gtpos 375 # find the name: w3.org/TR/html5/tokenization.html#tag-name-state 376 namematch = tagfind_tolerant.match(rawdata, i+2) 377 if not namematch: 378 # w3.org/TR/html5/tokenization.html#end-tag-open-state 379 if rawdata[i:i+3] == '</>': 380 return i+3 381 else: 382 return self.parse_bogus_comment(i) 383 tagname = namematch.group().lower() 384 # consume and ignore other stuff between the name and the > 385 # Note: this is not 100% correct, since we might have things like 386 # </tag attr=">">, but looking for > after tha name should cover 387 # most of the cases and is much simpler 388 gtpos = rawdata.find('>', namematch.end()) 389 self.handle_endtag(tagname) 390 return gtpos+1 391 392 elem = match.group(1).lower() # script or style 393 if self.cdata_elem is not None: 394 if elem != self.cdata_elem: 395 self.handle_data(rawdata[i:gtpos]) 396 return gtpos 397 398 self.handle_endtag(elem) 399 self.clear_cdata_mode() 400 return gtpos 401 402 # Overridable -- finish processing of start+end tag: <tag.../> 403 def handle_startendtag(self, tag, attrs): 404 self.handle_starttag(tag, attrs) 405 self.handle_endtag(tag) 406 407 # Overridable -- handle start tag 408 def handle_starttag(self, tag, attrs): 409 pass 410 411 # Overridable -- handle end tag 412 def handle_endtag(self, tag): 413 pass 414 415 # Overridable -- handle character reference 416 def handle_charref(self, name): 417 pass 418 419 # Overridable -- handle entity reference 420 def handle_entityref(self, name): 421 pass 422 423 # Overridable -- handle data 424 def handle_data(self, data): 425 pass 426 427 # Overridable -- handle comment 428 def handle_comment(self, data): 429 pass 430 431 # Overridable -- handle declaration 432 def handle_decl(self, decl): 433 pass 434 435 # Overridable -- handle processing instruction 436 def handle_pi(self, data): 437 pass 438 439 def unknown_decl(self, data): 440 pass 441 442 # Internal -- helper to remove special character quoting 443 entitydefs = None 444 def unescape(self, s): 445 if '&' not in s: 446 return s 447 def replaceEntities(s): 448 s = s.groups()[0] 449 try: 450 if s[0] == "#": 451 s = s[1:] 452 if s[0] in ['x','X']: 453 c = int(s[1:], 16) 454 else: 455 c = int(s) 456 return unichr(c) 457 except ValueError: 458 return '&#'+s+';' 459 else: 460 # Cannot use name2codepoint directly, because HTMLParser supports apos, 461 # which is not part of HTML 4 462 import htmlentitydefs 463 if HTMLParser.entitydefs is None: 464 entitydefs = HTMLParser.entitydefs = {'apos':u"'"} 465 for k, v in htmlentitydefs.name2codepoint.iteritems(): 466 entitydefs[k] = unichr(v) 467 try: 468 return self.entitydefs[s] 469 except KeyError: 470 return '&'+s+';' 471 472 return re.sub(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));", replaceEntities, s) 473
