Home | History | Annotate | Download | only in test 
      1 """Tests for HTMLParser.py."""
      2 
      3 import html.parser
      4 import pprint
      5 import unittest
      6 
      7 
      8 class EventCollector(html.parser.HTMLParser):
      9 
     10     def __init__(self, *args, **kw):
     11         self.events = []
     12         self.append = self.events.append
     13         html.parser.HTMLParser.__init__(self, *args, **kw)
     14 
     15     def get_events(self):
     16         # Normalize the list of events so that buffer artefacts don't
     17         # separate runs of contiguous characters.
     18         L = []
     19         prevtype = None
     20         for event in self.events:
     21             type = event[0]
     22             if type == prevtype == "data":
     23                 L[-1] = ("data", L[-1][1] + event[1])
     24             else:
     25                 L.append(event)
     26             prevtype = type
     27         self.events = L
     28         return L
     29 
     30     # structure markup
     31 
     32     def handle_starttag(self, tag, attrs):
     33         self.append(("starttag", tag, attrs))
     34 
     35     def handle_startendtag(self, tag, attrs):
     36         self.append(("startendtag", tag, attrs))
     37 
     38     def handle_endtag(self, tag):
     39         self.append(("endtag", tag))
     40 
     41     # all other markup
     42 
     43     def handle_comment(self, data):
     44         self.append(("comment", data))
     45 
     46     def handle_charref(self, data):
     47         self.append(("charref", data))
     48 
     49     def handle_data(self, data):
     50         self.append(("data", data))
     51 
     52     def handle_decl(self, data):
     53         self.append(("decl", data))
     54 
     55     def handle_entityref(self, data):
     56         self.append(("entityref", data))
     57 
     58     def handle_pi(self, data):
     59         self.append(("pi", data))
     60 
     61     def unknown_decl(self, decl):
     62         self.append(("unknown decl", decl))
     63 
     64 
     65 class EventCollectorExtra(EventCollector):
     66 
     67     def handle_starttag(self, tag, attrs):
     68         EventCollector.handle_starttag(self, tag, attrs)
     69         self.append(("starttag_text", self.get_starttag_text()))
     70 
     71 
     72 class EventCollectorCharrefs(EventCollector):
     73 
     74     def handle_charref(self, data):
     75         self.fail('This should never be called with convert_charrefs=True')
     76 
     77     def handle_entityref(self, data):
     78         self.fail('This should never be called with convert_charrefs=True')
     79 
     80 
     81 class TestCaseBase(unittest.TestCase):
     82 
     83     def get_collector(self):
     84         return EventCollector(convert_charrefs=False)
     85 
     86     def _run_check(self, source, expected_events, collector=None):
     87         if collector is None:
     88             collector = self.get_collector()
     89         parser = collector
     90         for s in source:
     91             parser.feed(s)
     92         parser.close()
     93         events = parser.get_events()
     94         if events != expected_events:
     95             self.fail("received events did not match expected events" +
     96                       "\nSource:\n" + repr(source) +
     97                       "\nExpected:\n" + pprint.pformat(expected_events) +
     98                       "\nReceived:\n" + pprint.pformat(events))
     99 
    100     def _run_check_extra(self, source, events):
    101         self._run_check(source, events,
    102                         EventCollectorExtra(convert_charrefs=False))
    103 
    104 
    105 class HTMLParserTestCase(TestCaseBase):
    106 
    107     def test_processing_instruction_only(self):
    108         self._run_check("<?processing instruction>", [
    109             ("pi", "processing instruction"),
    110             ])
    111         self._run_check("<?processing instruction ?>", [
    112             ("pi", "processing instruction ?"),
    113             ])
    114 
    115     def test_simple_html(self):
    116         self._run_check("""
    117 <!DOCTYPE html PUBLIC 'foo'>
    118 <HTML>&entity;&#32;
    119 <!--comment1a
    120 -></foo><bar>&lt;<?pi?></foo<bar
    121 comment1b-->
    122 <Img sRc='Bar' isMAP>sample
    123 text
    124 &#x201C;
    125 <!--comment2a-- --comment2b-->
    126 </Html>
    127 """, [
    128     ("data", "\n"),
    129     ("decl", "DOCTYPE html PUBLIC 'foo'"),
    130     ("data", "\n"),
    131     ("starttag", "html", []),
    132     ("entityref", "entity"),
    133     ("charref", "32"),
    134     ("data", "\n"),
    135     ("comment", "comment1a\n-></foo><bar>&lt;<?pi?></foo<bar\ncomment1b"),
    136     ("data", "\n"),
    137     ("starttag", "img", [("src", "Bar"), ("ismap", None)]),
    138     ("data", "sample\ntext\n"),
    139     ("charref", "x201C"),
    140     ("data", "\n"),
    141     ("comment", "comment2a-- --comment2b"),
    142     ("data", "\n"),
    143     ("endtag", "html"),
    144     ("data", "\n"),
    145     ])
    146 
    147     def test_malformatted_charref(self):
    148         self._run_check("<p>&#bad;</p>", [
    149             ("starttag", "p", []),
    150             ("data", "&#bad;"),
    151             ("endtag", "p"),
    152         ])
    153         # add the [] as a workaround to avoid buffering (see #20288)
    154         self._run_check(["<div>&#bad;</div>"], [
    155             ("starttag", "div", []),
    156             ("data", "&#bad;"),
    157             ("endtag", "div"),
    158         ])
    159 
    160     def test_unclosed_entityref(self):
    161         self._run_check("&entityref foo", [
    162             ("entityref", "entityref"),
    163             ("data", " foo"),
    164             ])
    165 
    166     def test_bad_nesting(self):
    167         # Strangely, this *is* supposed to test that overlapping
    168         # elements are allowed.  HTMLParser is more geared toward
    169         # lexing the input that parsing the structure.
    170         self._run_check("<a><b></a></b>", [
    171             ("starttag", "a", []),
    172             ("starttag", "b", []),
    173             ("endtag", "a"),
    174             ("endtag", "b"),
    175             ])
    176 
    177     def test_bare_ampersands(self):
    178         self._run_check("this text & contains & ampersands &", [
    179             ("data", "this text & contains & ampersands &"),
    180             ])
    181 
    182     def test_bare_pointy_brackets(self):
    183         self._run_check("this < text > contains < bare>pointy< brackets", [
    184             ("data", "this < text > contains < bare>pointy< brackets"),
    185             ])
    186 
    187     def test_starttag_end_boundary(self):
    188         self._run_check("""<a b='<'>""", [("starttag", "a", [("b", "<")])])
    189         self._run_check("""<a b='>'>""", [("starttag", "a", [("b", ">")])])
    190 
    191     def test_buffer_artefacts(self):
    192         output = [("starttag", "a", [("b", "<")])]
    193         self._run_check(["<a b='<'>"], output)
    194         self._run_check(["<a ", "b='<'>"], output)
    195         self._run_check(["<a b", "='<'>"], output)
    196         self._run_check(["<a b=", "'<'>"], output)
    197         self._run_check(["<a b='<", "'>"], output)
    198         self._run_check(["<a b='<'", ">"], output)
    199 
    200         output = [("starttag", "a", [("b", ">")])]
    201         self._run_check(["<a b='>'>"], output)
    202         self._run_check(["<a ", "b='>'>"], output)
    203         self._run_check(["<a b", "='>'>"], output)
    204         self._run_check(["<a b=", "'>'>"], output)
    205         self._run_check(["<a b='>", "'>"], output)
    206         self._run_check(["<a b='>'", ">"], output)
    207 
    208         output = [("comment", "abc")]
    209         self._run_check(["", "<!--abc-->"], output)
    210         self._run_check(["<", "!--abc-->"], output)
    211         self._run_check(["<!", "--abc-->"], output)
    212         self._run_check(["<!-", "-abc-->"], output)
    213         self._run_check(["<!--", "abc-->"], output)
    214         self._run_check(["<!--a", "bc-->"], output)
    215         self._run_check(["<!--ab", "c-->"], output)
    216         self._run_check(["<!--abc", "-->"], output)
    217         self._run_check(["<!--abc-", "->"], output)
    218         self._run_check(["<!--abc--", ">"], output)
    219         self._run_check(["<!--abc-->", ""], output)
    220 
    221     def test_valid_doctypes(self):
    222         # from http://www.w3.org/QA/2002/04/valid-dtd-list.html
    223         dtds = ['HTML',  # HTML5 doctype
    224                 ('HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
    225                  '"http://www.w3.org/TR/html4/strict.dtd"'),
    226                 ('HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" '
    227                  '"http://www.w3.org/TR/html4/loose.dtd"'),
    228                 ('html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" '
    229                  '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"'),
    230                 ('html PUBLIC "-//W3C//DTD XHTML 1.0 Frameset//EN" '
    231                  '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd"'),
    232                 ('math PUBLIC "-//W3C//DTD MathML 2.0//EN" '
    233                  '"http://www.w3.org/Math/DTD/mathml2/mathml2.dtd"'),
    234                 ('html PUBLIC "-//W3C//DTD '
    235                  'XHTML 1.1 plus MathML 2.0 plus SVG 1.1//EN" '
    236                  '"http://www.w3.org/2002/04/xhtml-math-svg/xhtml-math-svg.dtd"'),
    237                 ('svg PUBLIC "-//W3C//DTD SVG 1.1//EN" '
    238                  '"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"'),
    239                 'html PUBLIC "-//IETF//DTD HTML 2.0//EN"',
    240                 'html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"']
    241         for dtd in dtds:
    242             self._run_check("<!DOCTYPE %s>" % dtd,
    243                             [('decl', 'DOCTYPE ' + dtd)])
    244 
    245     def test_startendtag(self):
    246         self._run_check("<p/>", [
    247             ("startendtag", "p", []),
    248             ])
    249         self._run_check("<p></p>", [
    250             ("starttag", "p", []),
    251             ("endtag", "p"),
    252             ])
    253         self._run_check("<p><img src='foo' /></p>", [
    254             ("starttag", "p", []),
    255             ("startendtag", "img", [("src", "foo")]),
    256             ("endtag", "p"),
    257             ])
    258 
    259     def test_get_starttag_text(self):
    260         s = """<foo:bar   \n   one="1"\ttwo=2   >"""
    261         self._run_check_extra(s, [
    262             ("starttag", "foo:bar", [("one", "1"), ("two", "2")]),
    263             ("starttag_text", s)])
    264 
    265     def test_cdata_content(self):
    266         contents = [
    267             '<!-- not a comment --> &not-an-entity-ref;',
    268             "<not a='start tag'>",
    269             '<a href="" /> <p> <span></span>',
    270             'foo = "</scr" + "ipt>";',
    271             'foo = "</SCRIPT" + ">";',
    272             'foo = <\n/script> ',
    273             '<!-- document.write("</scr" + "ipt>"); -->',
    274             ('\n//<![CDATA[\n'
    275              'document.write(\'<s\'+\'cript type="text/javascript" '
    276              'src="http://www.example.org/r=\'+new '
    277              'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'),
    278             '\n<!-- //\nvar foo = 3.14;\n// -->\n',
    279             'foo = "</sty" + "le>";',
    280             '<!-- \u2603 -->',
    281             # these two should be invalid according to the HTML 5 spec,
    282             # section 8.1.2.2
    283             #'foo = </\nscript>',
    284             #'foo = </ script>',
    285         ]
    286         elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style']
    287         for content in contents:
    288             for element in elements:
    289                 element_lower = element.lower()
    290                 s = '<{element}>{content}</{element}>'.format(element=element,
    291                                                                content=content)
    292                 self._run_check(s, [("starttag", element_lower, []),
    293                                     ("data", content),
    294                                     ("endtag", element_lower)])
    295 
    296     def test_cdata_with_closing_tags(self):
    297         # see issue #13358
    298         # make sure that HTMLParser calls handle_data only once for each CDATA.
    299         # The normal event collector normalizes  the events in get_events,
    300         # so we override it to return the original list of events.
    301         class Collector(EventCollector):
    302             def get_events(self):
    303                 return self.events
    304 
    305         content = """<!-- not a comment --> &not-an-entity-ref;
    306                   <a href="" /> </p><p> <span></span></style>
    307                   '</script' + '>'"""
    308         for element in [' script', 'script ', ' script ',
    309                         '\nscript', 'script\n', '\nscript\n']:
    310             element_lower = element.lower().strip()
    311             s = '<script>{content}</{element}>'.format(element=element,
    312                                                        content=content)
    313             self._run_check(s, [("starttag", element_lower, []),
    314                                 ("data", content),
    315                                 ("endtag", element_lower)],
    316                             collector=Collector(convert_charrefs=False))
    317 
    318     def test_comments(self):
    319         html = ("<!-- I'm a valid comment -->"
    320                 '<!--me too!-->'
    321                 '<!------>'
    322                 '<!---->'
    323                 '<!----I have many hyphens---->'
    324                 '<!-- I have a > in the middle -->'
    325                 '<!-- and I have -- in the middle! -->')
    326         expected = [('comment', " I'm a valid comment "),
    327                     ('comment', 'me too!'),
    328                     ('comment', '--'),
    329                     ('comment', ''),
    330                     ('comment', '--I have many hyphens--'),
    331                     ('comment', ' I have a > in the middle '),
    332                     ('comment', ' and I have -- in the middle! ')]
    333         self._run_check(html, expected)
    334 
    335     def test_condcoms(self):
    336         html = ('<!--[if IE & !(lte IE 8)]>aren\'t<![endif]-->'
    337                 '<!--[if IE 8]>condcoms<![endif]-->'
    338                 '<!--[if lte IE 7]>pretty?<![endif]-->')
    339         expected = [('comment', "[if IE & !(lte IE 8)]>aren't<![endif]"),
    340                     ('comment', '[if IE 8]>condcoms<![endif]'),
    341                     ('comment', '[if lte IE 7]>pretty?<![endif]')]
    342         self._run_check(html, expected)
    343 
    344     def test_convert_charrefs(self):
    345         # default value for convert_charrefs is now True
    346         collector = lambda: EventCollectorCharrefs()
    347         self.assertTrue(collector().convert_charrefs)
    348         charrefs = ['&quot;', '&#34;', '&#x22;', '&quot', '&#34', '&#x22']
    349         # check charrefs in the middle of the text/attributes
    350         expected = [('starttag', 'a', [('href', 'foo"zar')]),
    351                     ('data', 'a"z'), ('endtag', 'a')]
    352         for charref in charrefs:
    353             self._run_check('<a href="foo{0}zar">a{0}z</a>'.format(charref),
    354                             expected, collector=collector())
    355         # check charrefs at the beginning/end of the text/attributes
    356         expected = [('data', '"'),
    357                     ('starttag', 'a', [('x', '"'), ('y', '"X'), ('z', 'X"')]),
    358                     ('data', '"'), ('endtag', 'a'), ('data', '"')]
    359         for charref in charrefs:
    360             self._run_check('{0}<a x="{0}" y="{0}X" z="X{0}">'
    361                             '{0}</a>{0}'.format(charref),
    362                             expected, collector=collector())
    363         # check charrefs in <script>/<style> elements
    364         for charref in charrefs:
    365             text = 'X'.join([charref]*3)
    366             expected = [('data', '"'),
    367                         ('starttag', 'script', []), ('data', text),
    368                         ('endtag', 'script'), ('data', '"'),
    369                         ('starttag', 'style', []), ('data', text),
    370                         ('endtag', 'style'), ('data', '"')]
    371             self._run_check('{1}<script>{0}</script>{1}'
    372                             '<style>{0}</style>{1}'.format(text, charref),
    373                             expected, collector=collector())
    374         # check truncated charrefs at the end of the file
    375         html = '&quo &# &#x'
    376         for x in range(1, len(html)):
    377             self._run_check(html[:x], [('data', html[:x])],
    378                             collector=collector())
    379         # check a string with no charrefs
    380         self._run_check('no charrefs here', [('data', 'no charrefs here')],
    381                         collector=collector())
    382 
    383     # the remaining tests were for the "tolerant" parser (which is now
    384     # the default), and check various kind of broken markup
    385     def test_tolerant_parsing(self):
    386         self._run_check('<html <html>te>>xt&a<<bc</a></html>\n'
    387                         '<img src="URL><//img></html</html>', [
    388                             ('starttag', 'html', [('<html', None)]),
    389                             ('data', 'te>>xt'),
    390                             ('entityref', 'a'),
    391                             ('data', '<'),
    392                             ('starttag', 'bc<', [('a', None)]),
    393                             ('endtag', 'html'),
    394                             ('data', '\n<img src="URL>'),
    395                             ('comment', '/img'),
    396                             ('endtag', 'html<')])
    397 
    398     def test_starttag_junk_chars(self):
    399         self._run_check("</>", [])
    400         self._run_check("</$>", [('comment', '$')])
    401         self._run_check("</", [('data', '</')])
    402         self._run_check("</a", [('data', '</a')])
    403         self._run_check("<a<a>", [('starttag', 'a<a', [])])
    404         self._run_check("</a<a>", [('endtag', 'a<a')])
    405         self._run_check("<!", [('data', '<!')])
    406         self._run_check("<a", [('data', '<a')])
    407         self._run_check("<a foo='bar'", [('data', "<a foo='bar'")])
    408         self._run_check("<a foo='bar", [('data', "<a foo='bar")])
    409         self._run_check("<a foo='>'", [('data', "<a foo='>'")])
    410         self._run_check("<a foo='>", [('data', "<a foo='>")])
    411         self._run_check("<a$>", [('starttag', 'a$', [])])
    412         self._run_check("<a$b>", [('starttag', 'a$b', [])])
    413         self._run_check("<a$b/>", [('startendtag', 'a$b', [])])
    414         self._run_check("<a$b  >", [('starttag', 'a$b', [])])
    415         self._run_check("<a$b  />", [('startendtag', 'a$b', [])])
    416 
    417     def test_slashes_in_starttag(self):
    418         self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])])
    419         html = ('<img width=902 height=250px '
    420                 'src="/sites/default/files/images/homepage/foo.jpg" '
    421                 '/*what am I doing here*/ />')
    422         expected = [(
    423             'startendtag', 'img',
    424             [('width', '902'), ('height', '250px'),
    425              ('src', '/sites/default/files/images/homepage/foo.jpg'),
    426              ('*what', None), ('am', None), ('i', None),
    427              ('doing', None), ('here*', None)]
    428         )]
    429         self._run_check(html, expected)
    430         html = ('<a / /foo/ / /=/ / /bar/ / />'
    431                 '<a / /foo/ / /=/ / /bar/ / >')
    432         expected = [
    433             ('startendtag', 'a', [('foo', None), ('=', None), ('bar', None)]),
    434             ('starttag', 'a', [('foo', None), ('=', None), ('bar', None)])
    435         ]
    436         self._run_check(html, expected)
    437         #see issue #14538
    438         html = ('<meta><meta / ><meta // ><meta / / >'
    439                 '<meta/><meta /><meta //><meta//>')
    440         expected = [
    441             ('starttag', 'meta', []), ('starttag', 'meta', []),
    442             ('starttag', 'meta', []), ('starttag', 'meta', []),
    443             ('startendtag', 'meta', []), ('startendtag', 'meta', []),
    444             ('startendtag', 'meta', []), ('startendtag', 'meta', []),
    445         ]
    446         self._run_check(html, expected)
    447 
    448     def test_declaration_junk_chars(self):
    449         self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')])
    450 
    451     def test_illegal_declarations(self):
    452         self._run_check('<!spacer type="block" height="25">',
    453                         [('comment', 'spacer type="block" height="25"')])
    454 
    455     def test_with_unquoted_attributes(self):
    456         # see #12008
    457         html = ("<html><body bgcolor=d0ca90 text='181008'>"
    458                 "<table cellspacing=0 cellpadding=1 width=100% ><tr>"
    459                 "<td align=left><font size=-1>"
    460                 "- <a href=/rabota/><span class=en> software-and-i</span></a>"
    461                 "- <a href='/1/'><span class=en> library</span></a></table>")
    462         expected = [
    463             ('starttag', 'html', []),
    464             ('starttag', 'body', [('bgcolor', 'd0ca90'), ('text', '181008')]),
    465             ('starttag', 'table',
    466                 [('cellspacing', '0'), ('cellpadding', '1'), ('width', '100%')]),
    467             ('starttag', 'tr', []),
    468             ('starttag', 'td', [('align', 'left')]),
    469             ('starttag', 'font', [('size', '-1')]),
    470             ('data', '- '), ('starttag', 'a', [('href', '/rabota/')]),
    471             ('starttag', 'span', [('class', 'en')]), ('data', ' software-and-i'),
    472             ('endtag', 'span'), ('endtag', 'a'),
    473             ('data', '- '), ('starttag', 'a', [('href', '/1/')]),
    474             ('starttag', 'span', [('class', 'en')]), ('data', ' library'),
    475             ('endtag', 'span'), ('endtag', 'a'), ('endtag', 'table')
    476         ]
    477         self._run_check(html, expected)
    478 
    479     def test_comma_between_attributes(self):
    480         self._run_check('<form action="/xxx.php?a=1&amp;b=2&amp", '
    481                         'method="post">', [
    482                             ('starttag', 'form',
    483                                 [('action', '/xxx.php?a=1&b=2&'),
    484                                  (',', None), ('method', 'post')])])
    485 
    486     def test_weird_chars_in_unquoted_attribute_values(self):
    487         self._run_check('<form action=bogus|&#()value>', [
    488                             ('starttag', 'form',
    489                                 [('action', 'bogus|&#()value')])])
    490 
    491     def test_invalid_end_tags(self):
    492         # A collection of broken end tags. <br> is used as separator.
    493         # see http://www.w3.org/TR/html5/tokenization.html#end-tag-open-state
    494         # and #13993
    495         html = ('<br></label</p><br></div end tmAd-leaderBoard><br></<h4><br>'
    496                 '</li class="unit"><br></li\r\n\t\t\t\t\t\t</ul><br></><br>')
    497         expected = [('starttag', 'br', []),
    498                     # < is part of the name, / is discarded, p is an attribute
    499                     ('endtag', 'label<'),
    500                     ('starttag', 'br', []),
    501                     # text and attributes are discarded
    502                     ('endtag', 'div'),
    503                     ('starttag', 'br', []),
    504                     # comment because the first char after </ is not a-zA-Z
    505                     ('comment', '<h4'),
    506                     ('starttag', 'br', []),
    507                     # attributes are discarded
    508                     ('endtag', 'li'),
    509                     ('starttag', 'br', []),
    510                     # everything till ul (included) is discarded
    511                     ('endtag', 'li'),
    512                     ('starttag', 'br', []),
    513                     # </> is ignored
    514                     ('starttag', 'br', [])]
    515         self._run_check(html, expected)
    516 
    517     def test_broken_invalid_end_tag(self):
    518         # This is technically wrong (the "> shouldn't be included in the 'data')
    519         # but is probably not worth fixing it (in addition to all the cases of
    520         # the previous test, it would require a full attribute parsing).
    521         # see #13993
    522         html = '<b>This</b attr=">"> confuses the parser'
    523         expected = [('starttag', 'b', []),
    524                     ('data', 'This'),
    525                     ('endtag', 'b'),
    526                     ('data', '"> confuses the parser')]
    527         self._run_check(html, expected)
    528 
    529     def test_correct_detection_of_start_tags(self):
    530         # see #13273
    531         html = ('<div style=""    ><b>The <a href="some_url">rain</a> '
    532                 '<br /> in <span>Spain</span></b></div>')
    533         expected = [
    534             ('starttag', 'div', [('style', '')]),
    535             ('starttag', 'b', []),
    536             ('data', 'The '),
    537             ('starttag', 'a', [('href', 'some_url')]),
    538             ('data', 'rain'),
    539             ('endtag', 'a'),
    540             ('data', ' '),
    541             ('startendtag', 'br', []),
    542             ('data', ' in '),
    543             ('starttag', 'span', []),
    544             ('data', 'Spain'),
    545             ('endtag', 'span'),
    546             ('endtag', 'b'),
    547             ('endtag', 'div')
    548         ]
    549         self._run_check(html, expected)
    550 
    551         html = '<div style="", foo = "bar" ><b>The <a href="some_url">rain</a>'
    552         expected = [
    553             ('starttag', 'div', [('style', ''), (',', None), ('foo', 'bar')]),
    554             ('starttag', 'b', []),
    555             ('data', 'The '),
    556             ('starttag', 'a', [('href', 'some_url')]),
    557             ('data', 'rain'),
    558             ('endtag', 'a'),
    559         ]
    560         self._run_check(html, expected)
    561 
    562     def test_EOF_in_charref(self):
    563         # see #17802
    564         # This test checks that the UnboundLocalError reported in the issue
    565         # is not raised, however I'm not sure the returned values are correct.
    566         # Maybe HTMLParser should use self.unescape for these
    567         data = [
    568             ('a&', [('data', 'a&')]),
    569             ('a&b', [('data', 'ab')]),
    570             ('a&b ', [('data', 'a'), ('entityref', 'b'), ('data', ' ')]),
    571             ('a&b;', [('data', 'a'), ('entityref', 'b')]),
    572         ]
    573         for html, expected in data:
    574             self._run_check(html, expected)
    575 
    576     def test_unescape_method(self):
    577         from html import unescape
    578         p = self.get_collector()
    579         with self.assertWarns(DeprecationWarning):
    580             s = '&quot;&#34;&#x22;&quot&#34&#x22&#bad;'
    581             self.assertEqual(p.unescape(s), unescape(s))
    582 
    583     def test_broken_comments(self):
    584         html = ('<! not really a comment >'
    585                 '<! not a comment either -->'
    586                 '<! -- close enough -->'
    587                 '<!><!<-- this was an empty comment>'
    588                 '<!!! another bogus comment !!!>')
    589         expected = [
    590             ('comment', ' not really a comment '),
    591             ('comment', ' not a comment either --'),
    592             ('comment', ' -- close enough --'),
    593             ('comment', ''),
    594             ('comment', '<-- this was an empty comment'),
    595             ('comment', '!! another bogus comment !!!'),
    596         ]
    597         self._run_check(html, expected)
    598 
    599     def test_broken_condcoms(self):
    600         # these condcoms are missing the '--' after '<!' and before the '>'
    601         html = ('<![if !(IE)]>broken condcom<![endif]>'
    602                 '<![if ! IE]><link href="favicon.tiff"/><![endif]>'
    603                 '<![if !IE 6]><img src="firefox.png" /><![endif]>'
    604                 '<![if !ie 6]><b>foo</b><![endif]>'
    605                 '<![if (!IE)|(lt IE 9)]><img src="mammoth.bmp" /><![endif]>')
    606         # According to the HTML5 specs sections "8.2.4.44 Bogus comment state"
    607         # and "8.2.4.45 Markup declaration open state", comment tokens should
    608         # be emitted instead of 'unknown decl', but calling unknown_decl
    609         # provides more flexibility.
    610         # See also Lib/_markupbase.py:parse_declaration
    611         expected = [
    612             ('unknown decl', 'if !(IE)'),
    613             ('data', 'broken condcom'),
    614             ('unknown decl', 'endif'),
    615             ('unknown decl', 'if ! IE'),
    616             ('startendtag', 'link', [('href', 'favicon.tiff')]),
    617             ('unknown decl', 'endif'),
    618             ('unknown decl', 'if !IE 6'),
    619             ('startendtag', 'img', [('src', 'firefox.png')]),
    620             ('unknown decl', 'endif'),
    621             ('unknown decl', 'if !ie 6'),
    622             ('starttag', 'b', []),
    623             ('data', 'foo'),
    624             ('endtag', 'b'),
    625             ('unknown decl', 'endif'),
    626             ('unknown decl', 'if (!IE)|(lt IE 9)'),
    627             ('startendtag', 'img', [('src', 'mammoth.bmp')]),
    628             ('unknown decl', 'endif')
    629         ]
    630         self._run_check(html, expected)
    631 
    632     def test_convert_charrefs_dropped_text(self):
    633         # #23144: make sure that all the events are triggered when
    634         # convert_charrefs is True, even if we don't call .close()
    635         parser = EventCollector(convert_charrefs=True)
    636         # before the fix, bar & baz was missing
    637         parser.feed("foo <a>link</a> bar &amp; baz")
    638         self.assertEqual(
    639             parser.get_events(),
    640             [('data', 'foo '), ('starttag', 'a', []), ('data', 'link'),
    641              ('endtag', 'a'), ('data', ' bar & baz')]
    642         )
    643 
    644 
    645 class AttributesTestCase(TestCaseBase):
    646 
    647     def test_attr_syntax(self):
    648         output = [
    649           ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)])
    650         ]
    651         self._run_check("""<a b='v' c="v" d=v e>""", output)
    652         self._run_check("""<a  b = 'v' c = "v" d = v e>""", output)
    653         self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output)
    654         self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output)
    655 
    656     def test_attr_values(self):
    657         self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""",
    658                         [("starttag", "a", [("b", "xxx\n\txxx"),
    659                                             ("c", "yyy\t\nyyy"),
    660                                             ("d", "\txyz\n")])])
    661         self._run_check("""<a b='' c="">""",
    662                         [("starttag", "a", [("b", ""), ("c", "")])])
    663         # Regression test for SF patch #669683.
    664         self._run_check("<e a=rgb(1,2,3)>",
    665                         [("starttag", "e", [("a", "rgb(1,2,3)")])])
    666         # Regression test for SF bug #921657.
    667         self._run_check(
    668             "<a href=mailto:xyz (at] example.com>",
    669             [("starttag", "a", [("href", "mailto:xyz (at] example.com")])])
    670 
    671     def test_attr_nonascii(self):
    672         # see issue 7311
    673         self._run_check(
    674             "<img src=/foo/bar.png alt=\u4e2d\u6587>",
    675             [("starttag", "img", [("src", "/foo/bar.png"),
    676                                   ("alt", "\u4e2d\u6587")])])
    677         self._run_check(
    678             "<a title='\u30c6\u30b9\u30c8' href='\u30c6\u30b9\u30c8.html'>",
    679             [("starttag", "a", [("title", "\u30c6\u30b9\u30c8"),
    680                                 ("href", "\u30c6\u30b9\u30c8.html")])])
    681         self._run_check(
    682             '<a title="\u30c6\u30b9\u30c8" href="\u30c6\u30b9\u30c8.html">',
    683             [("starttag", "a", [("title", "\u30c6\u30b9\u30c8"),
    684                                 ("href", "\u30c6\u30b9\u30c8.html")])])
    685 
    686     def test_attr_entity_replacement(self):
    687         self._run_check(
    688             "<a b='&amp;&gt;&lt;&quot;&apos;'>",
    689             [("starttag", "a", [("b", "&><\"'")])])
    690 
    691     def test_attr_funky_names(self):
    692         self._run_check(
    693             "<a a.b='v' c:d=v e-f=v>",
    694             [("starttag", "a", [("a.b", "v"), ("c:d", "v"), ("e-f", "v")])])
    695 
    696     def test_entityrefs_in_attributes(self):
    697         self._run_check(
    698             "<html foo='&euro;&amp;&#97;&#x61;&unsupported;'>",
    699             [("starttag", "html", [("foo", "\u20AC&aa&unsupported;")])])
    700 
    701 
    702     def test_attr_funky_names2(self):
    703         self._run_check(
    704             r"<a $><b $=%><c \=/>",
    705             [("starttag", "a", [("$", None)]),
    706              ("starttag", "b", [("$", "%")]),
    707              ("starttag", "c", [("\\", "/")])])
    708 
    709     def test_entities_in_attribute_value(self):
    710         # see #1200313
    711         for entity in ['&', '&amp;', '&#38;', '&#x26;']:
    712             self._run_check('<a href="%s">' % entity,
    713                             [("starttag", "a", [("href", "&")])])
    714             self._run_check("<a href='%s'>" % entity,
    715                             [("starttag", "a", [("href", "&")])])
    716             self._run_check("<a href=%s>" % entity,
    717                             [("starttag", "a", [("href", "&")])])
    718 
    719     def test_malformed_attributes(self):
    720         # see #13357
    721         html = (
    722             "<a href=test'style='color:red;bad1'>test - bad1</a>"
    723             "<a href=test'+style='color:red;ba2'>test - bad2</a>"
    724             "<a href=test'&nbsp;style='color:red;bad3'>test - bad3</a>"
    725             "<a href = test'&nbsp;style='color:red;bad4'  >test - bad4</a>"
    726         )
    727         expected = [
    728             ('starttag', 'a', [('href', "test'style='color:red;bad1'")]),
    729             ('data', 'test - bad1'), ('endtag', 'a'),
    730             ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]),
    731             ('data', 'test - bad2'), ('endtag', 'a'),
    732             ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]),
    733             ('data', 'test - bad3'), ('endtag', 'a'),
    734             ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]),
    735             ('data', 'test - bad4'), ('endtag', 'a')
    736         ]
    737         self._run_check(html, expected)
    738 
    739     def test_malformed_adjacent_attributes(self):
    740         # see #12629
    741         self._run_check('<x><y z=""o"" /></x>',
    742                         [('starttag', 'x', []),
    743                             ('startendtag', 'y', [('z', ''), ('o""', None)]),
    744                             ('endtag', 'x')])
    745         self._run_check('<x><y z="""" /></x>',
    746                         [('starttag', 'x', []),
    747                             ('startendtag', 'y', [('z', ''), ('""', None)]),
    748                             ('endtag', 'x')])
    749 
    750     # see #755670 for the following 3 tests
    751     def test_adjacent_attributes(self):
    752         self._run_check('<a width="100%"cellspacing=0>',
    753                         [("starttag", "a",
    754                           [("width", "100%"), ("cellspacing","0")])])
    755 
    756         self._run_check('<a id="foo"class="bar">',
    757                         [("starttag", "a",
    758                           [("id", "foo"), ("class","bar")])])
    759 
    760     def test_missing_attribute_value(self):
    761         self._run_check('<a v=>',
    762                         [("starttag", "a", [("v", "")])])
    763 
    764     def test_javascript_attribute_value(self):
    765         self._run_check("<a href=javascript:popup('/popup/help.html')>",
    766                         [("starttag", "a",
    767                           [("href", "javascript:popup('/popup/help.html')")])])
    768 
    769     def test_end_tag_in_attribute_value(self):
    770         # see #1745761
    771         self._run_check("<a href='http://www.example.org/\">;'>spam</a>",
    772                         [("starttag", "a",
    773                           [("href", "http://www.example.org/\">;")]),
    774                          ("data", "spam"), ("endtag", "a")])
    775 
    776 
    777 if __name__ == "__main__":
    778     unittest.main()
    779