Home | History | Annotate | Download | only in test 
      1 import pprint
      2 import re
      3 import unittest
      4 from test import test_support
      5 sgmllib = test_support.import_module('sgmllib', deprecated=True)
      6 
      7 
      8 class EventCollector(sgmllib.SGMLParser):
      9 
     10     def __init__(self):
     11         self.events = []
     12         self.append = self.events.append
     13         sgmllib.SGMLParser.__init__(self)
     14 
     15     def get_events(self):
     16         # Normalize the list of events so that buffer artefacts don't
     17         # separate runs of contiguous characters.
     18         L = []
     19         prevtype = None
     20         for event in self.events:
     21             type = event[0]
     22             if type == prevtype == "data":
     23                 L[-1] = ("data", L[-1][1] + event[1])
     24             else:
     25                 L.append(event)
     26             prevtype = type
     27         self.events = L
     28         return L
     29 
     30     # structure markup
     31 
     32     def unknown_starttag(self, tag, attrs):
     33         self.append(("starttag", tag, attrs))
     34 
     35     def unknown_endtag(self, tag):
     36         self.append(("endtag", tag))
     37 
     38     # all other markup
     39 
     40     def handle_comment(self, data):
     41         self.append(("comment", data))
     42 
     43     def handle_charref(self, data):
     44         self.append(("charref", data))
     45 
     46     def handle_data(self, data):
     47         self.append(("data", data))
     48 
     49     def handle_decl(self, decl):
     50         self.append(("decl", decl))
     51 
     52     def handle_entityref(self, data):
     53         self.append(("entityref", data))
     54 
     55     def handle_pi(self, data):
     56         self.append(("pi", data))
     57 
     58     def unknown_decl(self, decl):
     59         self.append(("unknown decl", decl))
     60 
     61 
     62 class CDATAEventCollector(EventCollector):
     63     def start_cdata(self, attrs):
     64         self.append(("starttag", "cdata", attrs))
     65         self.setliteral()
     66 
     67 
     68 class HTMLEntityCollector(EventCollector):
     69 
     70     entity_or_charref = re.compile('(?:&([a-zA-Z][-.a-zA-Z0-9]*)'
     71         '|&#(x[0-9a-zA-Z]+|[0-9]+))(;?)')
     72 
     73     def convert_charref(self, name):
     74         self.append(("charref", "convert", name))
     75         if name[0] != "x":
     76             return EventCollector.convert_charref(self, name)
     77 
     78     def convert_codepoint(self, codepoint):
     79         self.append(("codepoint", "convert", codepoint))
     80         EventCollector.convert_codepoint(self, codepoint)
     81 
     82     def convert_entityref(self, name):
     83         self.append(("entityref", "convert", name))
     84         return EventCollector.convert_entityref(self, name)
     85 
     86     # These to record that they were called, then pass the call along
     87     # to the default implementation so that it's actions can be
     88     # recorded.
     89 
     90     def handle_charref(self, data):
     91         self.append(("charref", data))
     92         sgmllib.SGMLParser.handle_charref(self, data)
     93 
     94     def handle_entityref(self, data):
     95         self.append(("entityref", data))
     96         sgmllib.SGMLParser.handle_entityref(self, data)
     97 
     98 
     99 class SGMLParserTestCase(unittest.TestCase):
    100 
    101     collector = EventCollector
    102 
    103     def get_events(self, source):
    104         parser = self.collector()
    105         try:
    106             for s in source:
    107                 parser.feed(s)
    108             parser.close()
    109         except:
    110             #self.events = parser.events
    111             raise
    112         return parser.get_events()
    113 
    114     def check_events(self, source, expected_events):
    115         try:
    116             events = self.get_events(source)
    117         except:
    118             #import sys
    119             #print >>sys.stderr, pprint.pformat(self.events)
    120             raise
    121         if events != expected_events:
    122             self.fail("received events did not match expected events\n"
    123                       "Expected:\n" + pprint.pformat(expected_events) +
    124                       "\nReceived:\n" + pprint.pformat(events))
    125 
    126     def check_parse_error(self, source):
    127         parser = EventCollector()
    128         try:
    129             parser.feed(source)
    130             parser.close()
    131         except sgmllib.SGMLParseError:
    132             pass
    133         else:
    134             self.fail("expected SGMLParseError for %r\nReceived:\n%s"
    135                       % (source, pprint.pformat(parser.get_events())))
    136 
    137     def test_doctype_decl_internal(self):
    138         inside = """\
    139 DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN'
    140              SYSTEM 'http://www.w3.org/TR/html401/strict.dtd' [
    141   <!ELEMENT html - O EMPTY>
    142   <!ATTLIST html
    143       version CDATA #IMPLIED
    144       profile CDATA 'DublinCore'>
    145   <!NOTATION datatype SYSTEM 'http://xml.python.org/notations/python-module'>
    146   <!ENTITY myEntity 'internal parsed entity'>
    147   <!ENTITY anEntity SYSTEM 'http://xml.python.org/entities/something.xml'>
    148   <!ENTITY % paramEntity 'name|name|name'>
    149   %paramEntity;
    150   <!-- comment -->
    151 ]"""
    152         self.check_events(["<!%s>" % inside], [
    153             ("decl", inside),
    154             ])
    155 
    156     def test_doctype_decl_external(self):
    157         inside = "DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN'"
    158         self.check_events("<!%s>" % inside, [
    159             ("decl", inside),
    160             ])
    161 
    162     def test_underscore_in_attrname(self):
    163         # SF bug #436621
    164         """Make sure attribute names with underscores are accepted"""
    165         self.check_events("<a has_under _under>", [
    166             ("starttag", "a", [("has_under", "has_under"),
    167                                ("_under", "_under")]),
    168             ])
    169 
    170     def test_underscore_in_tagname(self):
    171         # SF bug #436621
    172         """Make sure tag names with underscores are accepted"""
    173         self.check_events("<has_under></has_under>", [
    174             ("starttag", "has_under", []),
    175             ("endtag", "has_under"),
    176             ])
    177 
    178     def test_quotes_in_unquoted_attrs(self):
    179         # SF bug #436621
    180         """Be sure quotes in unquoted attributes are made part of the value"""
    181         self.check_events("<a href=foo'bar\"baz>", [
    182             ("starttag", "a", [("href", "foo'bar\"baz")]),
    183             ])
    184 
    185     def test_xhtml_empty_tag(self):
    186         """Handling of XHTML-style empty start tags"""
    187         self.check_events("<br />text<i></i>", [
    188             ("starttag", "br", []),
    189             ("data", "text"),
    190             ("starttag", "i", []),
    191             ("endtag", "i"),
    192             ])
    193 
    194     def test_processing_instruction_only(self):
    195         self.check_events("<?processing instruction>", [
    196             ("pi", "processing instruction"),
    197             ])
    198 
    199     def test_bad_nesting(self):
    200         self.check_events("<a><b></a></b>", [
    201             ("starttag", "a", []),
    202             ("starttag", "b", []),
    203             ("endtag", "a"),
    204             ("endtag", "b"),
    205             ])
    206 
    207     def test_bare_ampersands(self):
    208         self.check_events("this text & contains & ampersands &", [
    209             ("data", "this text & contains & ampersands &"),
    210             ])
    211 
    212     def test_bare_pointy_brackets(self):
    213         self.check_events("this < text > contains < bare>pointy< brackets", [
    214             ("data", "this < text > contains < bare>pointy< brackets"),
    215             ])
    216 
    217     def test_attr_syntax(self):
    218         output = [
    219           ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", "e")])
    220           ]
    221         self.check_events("""<a b='v' c="v" d=v e>""", output)
    222         self.check_events("""<a  b = 'v' c = "v" d = v e>""", output)
    223         self.check_events("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output)
    224         self.check_events("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output)
    225 
    226     def test_attr_values(self):
    227         self.check_events("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""",
    228                         [("starttag", "a", [("b", "xxx\n\txxx"),
    229                                             ("c", "yyy\t\nyyy"),
    230                                             ("d", "\txyz\n")])
    231                          ])
    232         self.check_events("""<a b='' c="">""", [
    233             ("starttag", "a", [("b", ""), ("c", "")]),
    234             ])
    235         # URL construction stuff from RFC 1808:
    236         safe = "$-_.+"
    237         extra = "!*'(),"
    238         reserved = ";/?:@&="
    239         url = "http://example.com:8080/path/to/file?%s%s%s" % (
    240             safe, extra, reserved)
    241         self.check_events("""<e a=%s>""" % url, [
    242             ("starttag", "e", [("a", url)]),
    243             ])
    244         # Regression test for SF patch #669683.
    245         self.check_events("<e a=rgb(1,2,3)>", [
    246             ("starttag", "e", [("a", "rgb(1,2,3)")]),
    247             ])
    248 
    249     def test_attr_values_entities(self):
    250         """Substitution of entities and charrefs in attribute values"""
    251         # SF bug #1452246
    252         self.check_events("""<a b=&lt; c=&lt;&gt; d=&lt-&gt; e='&lt; '
    253                                 f="&xxx;" g='&#32;&#33;' h='&#500;'
    254                                 i='x?a=b&c=d;'
    255                                 j='&amp;#42;' k='&#38;#42;'>""",
    256             [("starttag", "a", [("b", "<"),
    257                                 ("c", "<>"),
    258                                 ("d", "&lt->"),
    259                                 ("e", "< "),
    260                                 ("f", "&xxx;"),
    261                                 ("g", " !"),
    262                                 ("h", "&#500;"),
    263                                 ("i", "x?a=b&c=d;"),
    264                                 ("j", "&#42;"),
    265                                 ("k", "&#42;"),
    266                                 ])])
    267 
    268     def test_convert_overrides(self):
    269         # This checks that the character and entity reference
    270         # conversion helpers are called at the documented times.  No
    271         # attempt is made to really change what the parser accepts.
    272         #
    273         self.collector = HTMLEntityCollector
    274         self.check_events(('<a title="&ldquo;test&#x201d;">foo</a>'
    275                            '&foobar;&#42;'), [
    276             ('entityref', 'convert', 'ldquo'),
    277             ('charref', 'convert', 'x201d'),
    278             ('starttag', 'a', [('title', '&ldquo;test&#x201d;')]),
    279             ('data', 'foo'),
    280             ('endtag', 'a'),
    281             ('entityref', 'foobar'),
    282             ('entityref', 'convert', 'foobar'),
    283             ('charref', '42'),
    284             ('charref', 'convert', '42'),
    285             ('codepoint', 'convert', 42),
    286             ])
    287 
    288     def test_attr_funky_names(self):
    289         self.check_events("""<a a.b='v' c:d=v e-f=v>""", [
    290             ("starttag", "a", [("a.b", "v"), ("c:d", "v"), ("e-f", "v")]),
    291             ])
    292 
    293     def test_attr_value_ip6_url(self):
    294         # http://www.python.org/sf/853506
    295         self.check_events(("<a href='http://[1080::8:800:200C:417A]/'>"
    296                            "<a href=http://[1080::8:800:200C:417A]/>"), [
    297             ("starttag", "a", [("href", "http://[1080::8:800:200C:417A]/")]),
    298             ("starttag", "a", [("href", "http://[1080::8:800:200C:417A]/")]),
    299             ])
    300 
    301     def test_weird_starttags(self):
    302         self.check_events("<a<a>", [
    303             ("starttag", "a", []),
    304             ("starttag", "a", []),
    305             ])
    306         self.check_events("</a<a>", [
    307             ("endtag", "a"),
    308             ("starttag", "a", []),
    309             ])
    310 
    311     def test_declaration_junk_chars(self):
    312         self.check_parse_error("<!DOCTYPE foo $ >")
    313 
    314     def test_get_starttag_text(self):
    315         s = """<foobar   \n   one="1"\ttwo=2   >"""
    316         self.check_events(s, [
    317             ("starttag", "foobar", [("one", "1"), ("two", "2")]),
    318             ])
    319 
    320     def test_cdata_content(self):
    321         s = ("<cdata> <!-- not a comment --> &not-an-entity-ref; </cdata>"
    322              "<notcdata> <!-- comment --> </notcdata>")
    323         self.collector = CDATAEventCollector
    324         self.check_events(s, [
    325             ("starttag", "cdata", []),
    326             ("data", " <!-- not a comment --> &not-an-entity-ref; "),
    327             ("endtag", "cdata"),
    328             ("starttag", "notcdata", []),
    329             ("data", " "),
    330             ("comment", " comment "),
    331             ("data", " "),
    332             ("endtag", "notcdata"),
    333             ])
    334         s = """<cdata> <not a='start tag'> </cdata>"""
    335         self.check_events(s, [
    336             ("starttag", "cdata", []),
    337             ("data", " <not a='start tag'> "),
    338             ("endtag", "cdata"),
    339             ])
    340 
    341     def test_illegal_declarations(self):
    342         s = 'abc<!spacer type="block" height="25">def'
    343         self.check_events(s, [
    344             ("data", "abc"),
    345             ("unknown decl", 'spacer type="block" height="25"'),
    346             ("data", "def"),
    347             ])
    348 
    349     def test_enumerated_attr_type(self):
    350         s = "<!DOCTYPE doc [<!ATTLIST doc attr (a | b) >]>"
    351         self.check_events(s, [
    352             ('decl', 'DOCTYPE doc [<!ATTLIST doc attr (a | b) >]'),
    353             ])
    354 
    355     def test_read_chunks(self):
    356         # SF bug #1541697, this caused sgml parser to hang
    357         # Just verify this code doesn't cause a hang.
    358         CHUNK = 1024  # increasing this to 8212 makes the problem go away
    359 
    360         f = open(test_support.findfile('sgml_input.html'))
    361         fp = sgmllib.SGMLParser()
    362         while 1:
    363             data = f.read(CHUNK)
    364             fp.feed(data)
    365             if len(data) != CHUNK:
    366                 break
    367 
    368     def test_only_decode_ascii(self):
    369         # SF bug #1651995, make sure non-ascii character references are not decoded
    370         s = '<signs exclamation="&#33" copyright="&#169" quoteleft="&#8216;">'
    371         self.check_events(s, [
    372             ('starttag', 'signs',
    373              [('exclamation', '!'), ('copyright', '&#169'),
    374               ('quoteleft', '&#8216;')]),
    375             ])
    376 
    377     # XXX These tests have been disabled by prefixing their names with
    378     # an underscore.  The first two exercise outstanding bugs in the
    379     # sgmllib module, and the third exhibits questionable behavior
    380     # that needs to be carefully considered before changing it.
    381 
    382     def _test_starttag_end_boundary(self):
    383         self.check_events("<a b='<'>", [("starttag", "a", [("b", "<")])])
    384         self.check_events("<a b='>'>", [("starttag", "a", [("b", ">")])])
    385 
    386     def _test_buffer_artefacts(self):
    387         output = [("starttag", "a", [("b", "<")])]
    388         self.check_events(["<a b='<'>"], output)
    389         self.check_events(["<a ", "b='<'>"], output)
    390         self.check_events(["<a b", "='<'>"], output)
    391         self.check_events(["<a b=", "'<'>"], output)
    392         self.check_events(["<a b='<", "'>"], output)
    393         self.check_events(["<a b='<'", ">"], output)
    394 
    395         output = [("starttag", "a", [("b", ">")])]
    396         self.check_events(["<a b='>'>"], output)
    397         self.check_events(["<a ", "b='>'>"], output)
    398         self.check_events(["<a b", "='>'>"], output)
    399         self.check_events(["<a b=", "'>'>"], output)
    400         self.check_events(["<a b='>", "'>"], output)
    401         self.check_events(["<a b='>'", ">"], output)
    402 
    403         output = [("comment", "abc")]
    404         self.check_events(["", "<!--abc-->"], output)
    405         self.check_events(["<", "!--abc-->"], output)
    406         self.check_events(["<!", "--abc-->"], output)
    407         self.check_events(["<!-", "-abc-->"], output)
    408         self.check_events(["<!--", "abc-->"], output)
    409         self.check_events(["<!--a", "bc-->"], output)
    410         self.check_events(["<!--ab", "c-->"], output)
    411         self.check_events(["<!--abc", "-->"], output)
    412         self.check_events(["<!--abc-", "->"], output)
    413         self.check_events(["<!--abc--", ">"], output)
    414         self.check_events(["<!--abc-->", ""], output)
    415 
    416     def _test_starttag_junk_chars(self):
    417         self.check_parse_error("<")
    418         self.check_parse_error("<>")
    419         self.check_parse_error("</$>")
    420         self.check_parse_error("</")
    421         self.check_parse_error("</a")
    422         self.check_parse_error("<$")
    423         self.check_parse_error("<$>")
    424         self.check_parse_error("<!")
    425         self.check_parse_error("<a $>")
    426         self.check_parse_error("<a")
    427         self.check_parse_error("<a foo='bar'")
    428         self.check_parse_error("<a foo='bar")
    429         self.check_parse_error("<a foo='>'")
    430         self.check_parse_error("<a foo='>")
    431         self.check_parse_error("<a foo=>")
    432 
    433 
    434 def test_main():
    435     test_support.run_unittest(SGMLParserTestCase)
    436 
    437 
    438 if __name__ == "__main__":
    439     test_main()
    440