Home | History | Annotate | Download | only in Lib
      1 """HTML 2.0 parser.
      2 
      3 See the HTML 2.0 specification:
      4 http://www.w3.org/hypertext/WWW/MarkUp/html-spec/html-spec_toc.html
      5 """
      6 
      7 from warnings import warnpy3k
      8 warnpy3k("the htmllib module has been removed in Python 3.0",
      9          stacklevel=2)
     10 del warnpy3k
     11 
     12 import sgmllib
     13 
     14 from formatter import AS_IS
     15 
     16 __all__ = ["HTMLParser", "HTMLParseError"]
     17 
     18 
     19 class HTMLParseError(sgmllib.SGMLParseError):
     20     """Error raised when an HTML document can't be parsed."""
     21 
     22 
     23 class HTMLParser(sgmllib.SGMLParser):
     24     """This is the basic HTML parser class.
     25 
     26     It supports all entity names required by the XHTML 1.0 Recommendation.
     27     It also defines handlers for all HTML 2.0 and many HTML 3.0 and 3.2
     28     elements.
     29 
     30     """
     31 
     32     from htmlentitydefs import entitydefs
     33 
     34     def __init__(self, formatter, verbose=0):
     35         """Creates an instance of the HTMLParser class.
     36 
     37         The formatter parameter is the formatter instance associated with
     38         the parser.
     39 
     40         """
     41         sgmllib.SGMLParser.__init__(self, verbose)
     42         self.formatter = formatter
     43 
     44     def error(self, message):
     45         raise HTMLParseError(message)
     46 
     47     def reset(self):
     48         sgmllib.SGMLParser.reset(self)
     49         self.savedata = None
     50         self.isindex = 0
     51         self.title = None
     52         self.base = None
     53         self.anchor = None
     54         self.anchorlist = []
     55         self.nofill = 0
     56         self.list_stack = []
     57 
     58     # ------ Methods used internally; some may be overridden

     59 
     60     # --- Formatter interface, taking care of 'savedata' mode;

     61     # shouldn't need to be overridden

     62 
     63     def handle_data(self, data):
     64         if self.savedata is not None:
     65             self.savedata = self.savedata + data
     66         else:
     67             if self.nofill:
     68                 self.formatter.add_literal_data(data)
     69             else:
     70                 self.formatter.add_flowing_data(data)
     71 
     72     # --- Hooks to save data; shouldn't need to be overridden

     73 
     74     def save_bgn(self):
     75         """Begins saving character data in a buffer instead of sending it
     76         to the formatter object.
     77 
     78         Retrieve the stored data via the save_end() method.  Use of the
     79         save_bgn() / save_end() pair may not be nested.
     80 
     81         """
     82         self.savedata = ''
     83 
     84     def save_end(self):
     85         """Ends buffering character data and returns all data saved since
     86         the preceding call to the save_bgn() method.
     87 
     88         If the nofill flag is false, whitespace is collapsed to single
     89         spaces.  A call to this method without a preceding call to the
     90         save_bgn() method will raise a TypeError exception.
     91 
     92         """
     93         data = self.savedata
     94         self.savedata = None
     95         if not self.nofill:
     96             data = ' '.join(data.split())
     97         return data
     98 
     99     # --- Hooks for anchors; should probably be overridden

    100 
    101     def anchor_bgn(self, href, name, type):
    102         """This method is called at the start of an anchor region.
    103 
    104         The arguments correspond to the attributes of the <A> tag with
    105         the same names.  The default implementation maintains a list of
    106         hyperlinks (defined by the HREF attribute for <A> tags) within
    107         the document.  The list of hyperlinks is available as the data
    108         attribute anchorlist.
    109 
    110         """
    111         self.anchor = href
    112         if self.anchor:
    113             self.anchorlist.append(href)
    114 
    115     def anchor_end(self):
    116         """This method is called at the end of an anchor region.
    117 
    118         The default implementation adds a textual footnote marker using an
    119         index into the list of hyperlinks created by the anchor_bgn()method.
    120 
    121         """
    122         if self.anchor:
    123             self.handle_data("[%d]" % len(self.anchorlist))
    124             self.anchor = None
    125 
    126     # --- Hook for images; should probably be overridden

    127 
    128     def handle_image(self, src, alt, *args):
    129         """This method is called to handle images.
    130 
    131         The default implementation simply passes the alt value to the
    132         handle_data() method.
    133 
    134         """
    135         self.handle_data(alt)
    136 
    137     # --------- Top level elememts

    138 
    139     def start_html(self, attrs): pass
    140     def end_html(self): pass
    141 
    142     def start_head(self, attrs): pass
    143     def end_head(self): pass
    144 
    145     def start_body(self, attrs): pass
    146     def end_body(self): pass
    147 
    148     # ------ Head elements

    149 
    150     def start_title(self, attrs):
    151         self.save_bgn()
    152 
    153     def end_title(self):
    154         self.title = self.save_end()
    155 
    156     def do_base(self, attrs):
    157         for a, v in attrs:
    158             if a == 'href':
    159                 self.base = v
    160 
    161     def do_isindex(self, attrs):
    162         self.isindex = 1
    163 
    164     def do_link(self, attrs):
    165         pass
    166 
    167     def do_meta(self, attrs):
    168         pass
    169 
    170     def do_nextid(self, attrs): # Deprecated

    171         pass
    172 
    173     # ------ Body elements

    174 
    175     # --- Headings

    176 
    177     def start_h1(self, attrs):
    178         self.formatter.end_paragraph(1)
    179         self.formatter.push_font(('h1', 0, 1, 0))
    180 
    181     def end_h1(self):
    182         self.formatter.end_paragraph(1)
    183         self.formatter.pop_font()
    184 
    185     def start_h2(self, attrs):
    186         self.formatter.end_paragraph(1)
    187         self.formatter.push_font(('h2', 0, 1, 0))
    188 
    189     def end_h2(self):
    190         self.formatter.end_paragraph(1)
    191         self.formatter.pop_font()
    192 
    193     def start_h3(self, attrs):
    194         self.formatter.end_paragraph(1)
    195         self.formatter.push_font(('h3', 0, 1, 0))
    196 
    197     def end_h3(self):
    198         self.formatter.end_paragraph(1)
    199         self.formatter.pop_font()
    200 
    201     def start_h4(self, attrs):
    202         self.formatter.end_paragraph(1)
    203         self.formatter.push_font(('h4', 0, 1, 0))
    204 
    205     def end_h4(self):
    206         self.formatter.end_paragraph(1)
    207         self.formatter.pop_font()
    208 
    209     def start_h5(self, attrs):
    210         self.formatter.end_paragraph(1)
    211         self.formatter.push_font(('h5', 0, 1, 0))
    212 
    213     def end_h5(self):
    214         self.formatter.end_paragraph(1)
    215         self.formatter.pop_font()
    216 
    217     def start_h6(self, attrs):
    218         self.formatter.end_paragraph(1)
    219         self.formatter.push_font(('h6', 0, 1, 0))
    220 
    221     def end_h6(self):
    222         self.formatter.end_paragraph(1)
    223         self.formatter.pop_font()
    224 
    225     # --- Block Structuring Elements

    226 
    227     def do_p(self, attrs):
    228         self.formatter.end_paragraph(1)
    229 
    230     def start_pre(self, attrs):
    231         self.formatter.end_paragraph(1)
    232         self.formatter.push_font((AS_IS, AS_IS, AS_IS, 1))
    233         self.nofill = self.nofill + 1
    234 
    235     def end_pre(self):
    236         self.formatter.end_paragraph(1)
    237         self.formatter.pop_font()
    238         self.nofill = max(0, self.nofill - 1)
    239 
    240     def start_xmp(self, attrs):
    241         self.start_pre(attrs)
    242         self.setliteral('xmp') # Tell SGML parser

    243 
    244     def end_xmp(self):
    245         self.end_pre()
    246 
    247     def start_listing(self, attrs):
    248         self.start_pre(attrs)
    249         self.setliteral('listing') # Tell SGML parser

    250 
    251     def end_listing(self):
    252         self.end_pre()
    253 
    254     def start_address(self, attrs):
    255         self.formatter.end_paragraph(0)
    256         self.formatter.push_font((AS_IS, 1, AS_IS, AS_IS))
    257 
    258     def end_address(self):
    259         self.formatter.end_paragraph(0)
    260         self.formatter.pop_font()
    261 
    262     def start_blockquote(self, attrs):
    263         self.formatter.end_paragraph(1)
    264         self.formatter.push_margin('blockquote')
    265 
    266     def end_blockquote(self):
    267         self.formatter.end_paragraph(1)
    268         self.formatter.pop_margin()
    269 
    270     # --- List Elements

    271 
    272     def start_ul(self, attrs):
    273         self.formatter.end_paragraph(not self.list_stack)
    274         self.formatter.push_margin('ul')
    275         self.list_stack.append(['ul', '*', 0])
    276 
    277     def end_ul(self):
    278         if self.list_stack: del self.list_stack[-1]
    279         self.formatter.end_paragraph(not self.list_stack)
    280         self.formatter.pop_margin()
    281 
    282     def do_li(self, attrs):
    283         self.formatter.end_paragraph(0)
    284         if self.list_stack:
    285             [dummy, label, counter] = top = self.list_stack[-1]
    286             top[2] = counter = counter+1
    287         else:
    288             label, counter = '*', 0
    289         self.formatter.add_label_data(label, counter)
    290 
    291     def start_ol(self, attrs):
    292         self.formatter.end_paragraph(not self.list_stack)
    293         self.formatter.push_margin('ol')
    294         label = '1.'
    295         for a, v in attrs:
    296             if a == 'type':
    297                 if len(v) == 1: v = v + '.'
    298                 label = v
    299         self.list_stack.append(['ol', label, 0])
    300 
    301     def end_ol(self):
    302         if self.list_stack: del self.list_stack[-1]
    303         self.formatter.end_paragraph(not self.list_stack)
    304         self.formatter.pop_margin()
    305 
    306     def start_menu(self, attrs):
    307         self.start_ul(attrs)
    308 
    309     def end_menu(self):
    310         self.end_ul()
    311 
    312     def start_dir(self, attrs):
    313         self.start_ul(attrs)
    314 
    315     def end_dir(self):
    316         self.end_ul()
    317 
    318     def start_dl(self, attrs):
    319         self.formatter.end_paragraph(1)
    320         self.list_stack.append(['dl', '', 0])
    321 
    322     def end_dl(self):
    323         self.ddpop(1)
    324         if self.list_stack: del self.list_stack[-1]
    325 
    326     def do_dt(self, attrs):
    327         self.ddpop()
    328 
    329     def do_dd(self, attrs):
    330         self.ddpop()
    331         self.formatter.push_margin('dd')
    332         self.list_stack.append(['dd', '', 0])
    333 
    334     def ddpop(self, bl=0):
    335         self.formatter.end_paragraph(bl)
    336         if self.list_stack:
    337             if self.list_stack[-1][0] == 'dd':
    338                 del self.list_stack[-1]
    339                 self.formatter.pop_margin()
    340 
    341     # --- Phrase Markup

    342 
    343     # Idiomatic Elements

    344 
    345     def start_cite(self, attrs): self.start_i(attrs)
    346     def end_cite(self): self.end_i()
    347 
    348     def start_code(self, attrs): self.start_tt(attrs)
    349     def end_code(self): self.end_tt()
    350 
    351     def start_em(self, attrs): self.start_i(attrs)
    352     def end_em(self): self.end_i()
    353 
    354     def start_kbd(self, attrs): self.start_tt(attrs)
    355     def end_kbd(self): self.end_tt()
    356 
    357     def start_samp(self, attrs): self.start_tt(attrs)
    358     def end_samp(self): self.end_tt()
    359 
    360     def start_strong(self, attrs): self.start_b(attrs)
    361     def end_strong(self): self.end_b()
    362 
    363     def start_var(self, attrs): self.start_i(attrs)
    364     def end_var(self): self.end_i()
    365 
    366     # Typographic Elements

    367 
    368     def start_i(self, attrs):
    369         self.formatter.push_font((AS_IS, 1, AS_IS, AS_IS))
    370     def end_i(self):
    371         self.formatter.pop_font()
    372 
    373     def start_b(self, attrs):
    374         self.formatter.push_font((AS_IS, AS_IS, 1, AS_IS))
    375     def end_b(self):
    376         self.formatter.pop_font()
    377 
    378     def start_tt(self, attrs):
    379         self.formatter.push_font((AS_IS, AS_IS, AS_IS, 1))
    380     def end_tt(self):
    381         self.formatter.pop_font()
    382 
    383     def start_a(self, attrs):
    384         href = ''
    385         name = ''
    386         type = ''
    387         for attrname, value in attrs:
    388             value = value.strip()
    389             if attrname == 'href':
    390                 href = value
    391             if attrname == 'name':
    392                 name = value
    393             if attrname == 'type':
    394                 type = value.lower()
    395         self.anchor_bgn(href, name, type)
    396 
    397     def end_a(self):
    398         self.anchor_end()
    399 
    400     # --- Line Break

    401 
    402     def do_br(self, attrs):
    403         self.formatter.add_line_break()
    404 
    405     # --- Horizontal Rule

    406 
    407     def do_hr(self, attrs):
    408         self.formatter.add_hor_rule()
    409 
    410     # --- Image

    411 
    412     def do_img(self, attrs):
    413         align = ''
    414         alt = '(image)'
    415         ismap = ''
    416         src = ''
    417         width = 0
    418         height = 0
    419         for attrname, value in attrs:
    420             if attrname == 'align':
    421                 align = value
    422             if attrname == 'alt':
    423                 alt = value
    424             if attrname == 'ismap':
    425                 ismap = value
    426             if attrname == 'src':
    427                 src = value
    428             if attrname == 'width':
    429                 try: width = int(value)
    430                 except ValueError: pass
    431             if attrname == 'height':
    432                 try: height = int(value)
    433                 except ValueError: pass
    434         self.handle_image(src, alt, ismap, align, width, height)
    435 
    436     # --- Really Old Unofficial Deprecated Stuff

    437 
    438     def do_plaintext(self, attrs):
    439         self.start_pre(attrs)
    440         self.setnomoretags() # Tell SGML parser

    441 
    442     # --- Unhandled tags

    443 
    444     def unknown_starttag(self, tag, attrs):
    445         pass
    446 
    447     def unknown_endtag(self, tag):
    448         pass
    449 
    450 
    451 def test(args = None):
    452     import sys, formatter
    453 
    454     if not args:
    455         args = sys.argv[1:]
    456 
    457     silent = args and args[0] == '-s'
    458     if silent:
    459         del args[0]
    460 
    461     if args:
    462         file = args[0]
    463     else:
    464         file = 'test.html'
    465 
    466     if file == '-':
    467         f = sys.stdin
    468     else:
    469         try:
    470             f = open(file, 'r')
    471         except IOError, msg:
    472             print file, ":", msg
    473             sys.exit(1)
    474 
    475     data = f.read()
    476 
    477     if f is not sys.stdin:
    478         f.close()
    479 
    480     if silent:
    481         f = formatter.NullFormatter()
    482     else:
    483         f = formatter.AbstractFormatter(formatter.DumbWriter())
    484 
    485     p = HTMLParser(f)
    486     p.feed(data)
    487     p.close()
    488 
    489 
    490 if __name__ == '__main__':
    491     test()
    492