1 """HTML 2.0 parser. 2 3 See the HTML 2.0 specification: 4 http://www.w3.org/hypertext/WWW/MarkUp/html-spec/html-spec_toc.html 5 """ 6 7 from warnings import warnpy3k 8 warnpy3k("the htmllib module has been removed in Python 3.0", 9 stacklevel=2) 10 del warnpy3k 11 12 import sgmllib 13 14 from formatter import AS_IS 15 16 __all__ = ["HTMLParser", "HTMLParseError"] 17 18 19 class HTMLParseError(sgmllib.SGMLParseError): 20 """Error raised when an HTML document can't be parsed.""" 21 22 23 class HTMLParser(sgmllib.SGMLParser): 24 """This is the basic HTML parser class. 25 26 It supports all entity names required by the XHTML 1.0 Recommendation. 27 It also defines handlers for all HTML 2.0 and many HTML 3.0 and 3.2 28 elements. 29 30 """ 31 32 from htmlentitydefs import entitydefs 33 34 def __init__(self, formatter, verbose=0): 35 """Creates an instance of the HTMLParser class. 36 37 The formatter parameter is the formatter instance associated with 38 the parser. 39 40 """ 41 sgmllib.SGMLParser.__init__(self, verbose) 42 self.formatter = formatter 43 44 def error(self, message): 45 raise HTMLParseError(message) 46 47 def reset(self): 48 sgmllib.SGMLParser.reset(self) 49 self.savedata = None 50 self.isindex = 0 51 self.title = None 52 self.base = None 53 self.anchor = None 54 self.anchorlist = [] 55 self.nofill = 0 56 self.list_stack = [] 57 58 # ------ Methods used internally; some may be overridden 59 60 # --- Formatter interface, taking care of 'savedata' mode; 61 # shouldn't need to be overridden 62 63 def handle_data(self, data): 64 if self.savedata is not None: 65 self.savedata = self.savedata + data 66 else: 67 if self.nofill: 68 self.formatter.add_literal_data(data) 69 else: 70 self.formatter.add_flowing_data(data) 71 72 # --- Hooks to save data; shouldn't need to be overridden 73 74 def save_bgn(self): 75 """Begins saving character data in a buffer instead of sending it 76 to the formatter object. 77 78 Retrieve the stored data via the save_end() method. Use of the 79 save_bgn() / save_end() pair may not be nested. 80 81 """ 82 self.savedata = '' 83 84 def save_end(self): 85 """Ends buffering character data and returns all data saved since 86 the preceding call to the save_bgn() method. 87 88 If the nofill flag is false, whitespace is collapsed to single 89 spaces. A call to this method without a preceding call to the 90 save_bgn() method will raise a TypeError exception. 91 92 """ 93 data = self.savedata 94 self.savedata = None 95 if not self.nofill: 96 data = ' '.join(data.split()) 97 return data 98 99 # --- Hooks for anchors; should probably be overridden 100 101 def anchor_bgn(self, href, name, type): 102 """This method is called at the start of an anchor region. 103 104 The arguments correspond to the attributes of the <A> tag with 105 the same names. The default implementation maintains a list of 106 hyperlinks (defined by the HREF attribute for <A> tags) within 107 the document. The list of hyperlinks is available as the data 108 attribute anchorlist. 109 110 """ 111 self.anchor = href 112 if self.anchor: 113 self.anchorlist.append(href) 114 115 def anchor_end(self): 116 """This method is called at the end of an anchor region. 117 118 The default implementation adds a textual footnote marker using an 119 index into the list of hyperlinks created by the anchor_bgn()method. 120 121 """ 122 if self.anchor: 123 self.handle_data("[%d]" % len(self.anchorlist)) 124 self.anchor = None 125 126 # --- Hook for images; should probably be overridden 127 128 def handle_image(self, src, alt, *args): 129 """This method is called to handle images. 130 131 The default implementation simply passes the alt value to the 132 handle_data() method. 133 134 """ 135 self.handle_data(alt) 136 137 # --------- Top level elememts 138 139 def start_html(self, attrs): pass 140 def end_html(self): pass 141 142 def start_head(self, attrs): pass 143 def end_head(self): pass 144 145 def start_body(self, attrs): pass 146 def end_body(self): pass 147 148 # ------ Head elements 149 150 def start_title(self, attrs): 151 self.save_bgn() 152 153 def end_title(self): 154 self.title = self.save_end() 155 156 def do_base(self, attrs): 157 for a, v in attrs: 158 if a == 'href': 159 self.base = v 160 161 def do_isindex(self, attrs): 162 self.isindex = 1 163 164 def do_link(self, attrs): 165 pass 166 167 def do_meta(self, attrs): 168 pass 169 170 def do_nextid(self, attrs): # Deprecated 171 pass 172 173 # ------ Body elements 174 175 # --- Headings 176 177 def start_h1(self, attrs): 178 self.formatter.end_paragraph(1) 179 self.formatter.push_font(('h1', 0, 1, 0)) 180 181 def end_h1(self): 182 self.formatter.end_paragraph(1) 183 self.formatter.pop_font() 184 185 def start_h2(self, attrs): 186 self.formatter.end_paragraph(1) 187 self.formatter.push_font(('h2', 0, 1, 0)) 188 189 def end_h2(self): 190 self.formatter.end_paragraph(1) 191 self.formatter.pop_font() 192 193 def start_h3(self, attrs): 194 self.formatter.end_paragraph(1) 195 self.formatter.push_font(('h3', 0, 1, 0)) 196 197 def end_h3(self): 198 self.formatter.end_paragraph(1) 199 self.formatter.pop_font() 200 201 def start_h4(self, attrs): 202 self.formatter.end_paragraph(1) 203 self.formatter.push_font(('h4', 0, 1, 0)) 204 205 def end_h4(self): 206 self.formatter.end_paragraph(1) 207 self.formatter.pop_font() 208 209 def start_h5(self, attrs): 210 self.formatter.end_paragraph(1) 211 self.formatter.push_font(('h5', 0, 1, 0)) 212 213 def end_h5(self): 214 self.formatter.end_paragraph(1) 215 self.formatter.pop_font() 216 217 def start_h6(self, attrs): 218 self.formatter.end_paragraph(1) 219 self.formatter.push_font(('h6', 0, 1, 0)) 220 221 def end_h6(self): 222 self.formatter.end_paragraph(1) 223 self.formatter.pop_font() 224 225 # --- Block Structuring Elements 226 227 def do_p(self, attrs): 228 self.formatter.end_paragraph(1) 229 230 def start_pre(self, attrs): 231 self.formatter.end_paragraph(1) 232 self.formatter.push_font((AS_IS, AS_IS, AS_IS, 1)) 233 self.nofill = self.nofill + 1 234 235 def end_pre(self): 236 self.formatter.end_paragraph(1) 237 self.formatter.pop_font() 238 self.nofill = max(0, self.nofill - 1) 239 240 def start_xmp(self, attrs): 241 self.start_pre(attrs) 242 self.setliteral('xmp') # Tell SGML parser 243 244 def end_xmp(self): 245 self.end_pre() 246 247 def start_listing(self, attrs): 248 self.start_pre(attrs) 249 self.setliteral('listing') # Tell SGML parser 250 251 def end_listing(self): 252 self.end_pre() 253 254 def start_address(self, attrs): 255 self.formatter.end_paragraph(0) 256 self.formatter.push_font((AS_IS, 1, AS_IS, AS_IS)) 257 258 def end_address(self): 259 self.formatter.end_paragraph(0) 260 self.formatter.pop_font() 261 262 def start_blockquote(self, attrs): 263 self.formatter.end_paragraph(1) 264 self.formatter.push_margin('blockquote') 265 266 def end_blockquote(self): 267 self.formatter.end_paragraph(1) 268 self.formatter.pop_margin() 269 270 # --- List Elements 271 272 def start_ul(self, attrs): 273 self.formatter.end_paragraph(not self.list_stack) 274 self.formatter.push_margin('ul') 275 self.list_stack.append(['ul', '*', 0]) 276 277 def end_ul(self): 278 if self.list_stack: del self.list_stack[-1] 279 self.formatter.end_paragraph(not self.list_stack) 280 self.formatter.pop_margin() 281 282 def do_li(self, attrs): 283 self.formatter.end_paragraph(0) 284 if self.list_stack: 285 [dummy, label, counter] = top = self.list_stack[-1] 286 top[2] = counter = counter+1 287 else: 288 label, counter = '*', 0 289 self.formatter.add_label_data(label, counter) 290 291 def start_ol(self, attrs): 292 self.formatter.end_paragraph(not self.list_stack) 293 self.formatter.push_margin('ol') 294 label = '1.' 295 for a, v in attrs: 296 if a == 'type': 297 if len(v) == 1: v = v + '.' 298 label = v 299 self.list_stack.append(['ol', label, 0]) 300 301 def end_ol(self): 302 if self.list_stack: del self.list_stack[-1] 303 self.formatter.end_paragraph(not self.list_stack) 304 self.formatter.pop_margin() 305 306 def start_menu(self, attrs): 307 self.start_ul(attrs) 308 309 def end_menu(self): 310 self.end_ul() 311 312 def start_dir(self, attrs): 313 self.start_ul(attrs) 314 315 def end_dir(self): 316 self.end_ul() 317 318 def start_dl(self, attrs): 319 self.formatter.end_paragraph(1) 320 self.list_stack.append(['dl', '', 0]) 321 322 def end_dl(self): 323 self.ddpop(1) 324 if self.list_stack: del self.list_stack[-1] 325 326 def do_dt(self, attrs): 327 self.ddpop() 328 329 def do_dd(self, attrs): 330 self.ddpop() 331 self.formatter.push_margin('dd') 332 self.list_stack.append(['dd', '', 0]) 333 334 def ddpop(self, bl=0): 335 self.formatter.end_paragraph(bl) 336 if self.list_stack: 337 if self.list_stack[-1][0] == 'dd': 338 del self.list_stack[-1] 339 self.formatter.pop_margin() 340 341 # --- Phrase Markup 342 343 # Idiomatic Elements 344 345 def start_cite(self, attrs): self.start_i(attrs) 346 def end_cite(self): self.end_i() 347 348 def start_code(self, attrs): self.start_tt(attrs) 349 def end_code(self): self.end_tt() 350 351 def start_em(self, attrs): self.start_i(attrs) 352 def end_em(self): self.end_i() 353 354 def start_kbd(self, attrs): self.start_tt(attrs) 355 def end_kbd(self): self.end_tt() 356 357 def start_samp(self, attrs): self.start_tt(attrs) 358 def end_samp(self): self.end_tt() 359 360 def start_strong(self, attrs): self.start_b(attrs) 361 def end_strong(self): self.end_b() 362 363 def start_var(self, attrs): self.start_i(attrs) 364 def end_var(self): self.end_i() 365 366 # Typographic Elements 367 368 def start_i(self, attrs): 369 self.formatter.push_font((AS_IS, 1, AS_IS, AS_IS)) 370 def end_i(self): 371 self.formatter.pop_font() 372 373 def start_b(self, attrs): 374 self.formatter.push_font((AS_IS, AS_IS, 1, AS_IS)) 375 def end_b(self): 376 self.formatter.pop_font() 377 378 def start_tt(self, attrs): 379 self.formatter.push_font((AS_IS, AS_IS, AS_IS, 1)) 380 def end_tt(self): 381 self.formatter.pop_font() 382 383 def start_a(self, attrs): 384 href = '' 385 name = '' 386 type = '' 387 for attrname, value in attrs: 388 value = value.strip() 389 if attrname == 'href': 390 href = value 391 if attrname == 'name': 392 name = value 393 if attrname == 'type': 394 type = value.lower() 395 self.anchor_bgn(href, name, type) 396 397 def end_a(self): 398 self.anchor_end() 399 400 # --- Line Break 401 402 def do_br(self, attrs): 403 self.formatter.add_line_break() 404 405 # --- Horizontal Rule 406 407 def do_hr(self, attrs): 408 self.formatter.add_hor_rule() 409 410 # --- Image 411 412 def do_img(self, attrs): 413 align = '' 414 alt = '(image)' 415 ismap = '' 416 src = '' 417 width = 0 418 height = 0 419 for attrname, value in attrs: 420 if attrname == 'align': 421 align = value 422 if attrname == 'alt': 423 alt = value 424 if attrname == 'ismap': 425 ismap = value 426 if attrname == 'src': 427 src = value 428 if attrname == 'width': 429 try: width = int(value) 430 except ValueError: pass 431 if attrname == 'height': 432 try: height = int(value) 433 except ValueError: pass 434 self.handle_image(src, alt, ismap, align, width, height) 435 436 # --- Really Old Unofficial Deprecated Stuff 437 438 def do_plaintext(self, attrs): 439 self.start_pre(attrs) 440 self.setnomoretags() # Tell SGML parser 441 442 # --- Unhandled tags 443 444 def unknown_starttag(self, tag, attrs): 445 pass 446 447 def unknown_endtag(self, tag): 448 pass 449 450 451 def test(args = None): 452 import sys, formatter 453 454 if not args: 455 args = sys.argv[1:] 456 457 silent = args and args[0] == '-s' 458 if silent: 459 del args[0] 460 461 if args: 462 file = args[0] 463 else: 464 file = 'test.html' 465 466 if file == '-': 467 f = sys.stdin 468 else: 469 try: 470 f = open(file, 'r') 471 except IOError, msg: 472 print file, ":", msg 473 sys.exit(1) 474 475 data = f.read() 476 477 if f is not sys.stdin: 478 f.close() 479 480 if silent: 481 f = formatter.NullFormatter() 482 else: 483 f = formatter.AbstractFormatter(formatter.DumbWriter()) 484 485 p = HTMLParser(f) 486 p.feed(data) 487 p.close() 488 489 490 if __name__ == '__main__': 491 test() 492
