1 import pprint 2 import re 3 import unittest 4 from test import test_support 5 sgmllib = test_support.import_module('sgmllib', deprecated=True) 6 7 8 class EventCollector(sgmllib.SGMLParser): 9 10 def __init__(self): 11 self.events = [] 12 self.append = self.events.append 13 sgmllib.SGMLParser.__init__(self) 14 15 def get_events(self): 16 # Normalize the list of events so that buffer artefacts don't 17 # separate runs of contiguous characters. 18 L = [] 19 prevtype = None 20 for event in self.events: 21 type = event[0] 22 if type == prevtype == "data": 23 L[-1] = ("data", L[-1][1] + event[1]) 24 else: 25 L.append(event) 26 prevtype = type 27 self.events = L 28 return L 29 30 # structure markup 31 32 def unknown_starttag(self, tag, attrs): 33 self.append(("starttag", tag, attrs)) 34 35 def unknown_endtag(self, tag): 36 self.append(("endtag", tag)) 37 38 # all other markup 39 40 def handle_comment(self, data): 41 self.append(("comment", data)) 42 43 def handle_charref(self, data): 44 self.append(("charref", data)) 45 46 def handle_data(self, data): 47 self.append(("data", data)) 48 49 def handle_decl(self, decl): 50 self.append(("decl", decl)) 51 52 def handle_entityref(self, data): 53 self.append(("entityref", data)) 54 55 def handle_pi(self, data): 56 self.append(("pi", data)) 57 58 def unknown_decl(self, decl): 59 self.append(("unknown decl", decl)) 60 61 62 class CDATAEventCollector(EventCollector): 63 def start_cdata(self, attrs): 64 self.append(("starttag", "cdata", attrs)) 65 self.setliteral() 66 67 68 class HTMLEntityCollector(EventCollector): 69 70 entity_or_charref = re.compile('(?:&([a-zA-Z][-.a-zA-Z0-9]*)' 71 '|&#(x[0-9a-zA-Z]+|[0-9]+))(;?)') 72 73 def convert_charref(self, name): 74 self.append(("charref", "convert", name)) 75 if name[0] != "x": 76 return EventCollector.convert_charref(self, name) 77 78 def convert_codepoint(self, codepoint): 79 self.append(("codepoint", "convert", codepoint)) 80 EventCollector.convert_codepoint(self, codepoint) 81 82 def convert_entityref(self, name): 83 self.append(("entityref", "convert", name)) 84 return EventCollector.convert_entityref(self, name) 85 86 # These to record that they were called, then pass the call along 87 # to the default implementation so that it's actions can be 88 # recorded. 89 90 def handle_charref(self, data): 91 self.append(("charref", data)) 92 sgmllib.SGMLParser.handle_charref(self, data) 93 94 def handle_entityref(self, data): 95 self.append(("entityref", data)) 96 sgmllib.SGMLParser.handle_entityref(self, data) 97 98 99 class SGMLParserTestCase(unittest.TestCase): 100 101 collector = EventCollector 102 103 def get_events(self, source): 104 parser = self.collector() 105 try: 106 for s in source: 107 parser.feed(s) 108 parser.close() 109 except: 110 #self.events = parser.events 111 raise 112 return parser.get_events() 113 114 def check_events(self, source, expected_events): 115 try: 116 events = self.get_events(source) 117 except: 118 #import sys 119 #print >>sys.stderr, pprint.pformat(self.events) 120 raise 121 if events != expected_events: 122 self.fail("received events did not match expected events\n" 123 "Expected:\n" + pprint.pformat(expected_events) + 124 "\nReceived:\n" + pprint.pformat(events)) 125 126 def check_parse_error(self, source): 127 parser = EventCollector() 128 try: 129 parser.feed(source) 130 parser.close() 131 except sgmllib.SGMLParseError: 132 pass 133 else: 134 self.fail("expected SGMLParseError for %r\nReceived:\n%s" 135 % (source, pprint.pformat(parser.get_events()))) 136 137 def test_doctype_decl_internal(self): 138 inside = """\ 139 DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN' 140 SYSTEM 'http://www.w3.org/TR/html401/strict.dtd' [ 141 <!ELEMENT html - O EMPTY> 142 <!ATTLIST html 143 version CDATA #IMPLIED 144 profile CDATA 'DublinCore'> 145 <!NOTATION datatype SYSTEM 'http://xml.python.org/notations/python-module'> 146 <!ENTITY myEntity 'internal parsed entity'> 147 <!ENTITY anEntity SYSTEM 'http://xml.python.org/entities/something.xml'> 148 <!ENTITY % paramEntity 'name|name|name'> 149 %paramEntity; 150 <!-- comment --> 151 ]""" 152 self.check_events(["<!%s>" % inside], [ 153 ("decl", inside), 154 ]) 155 156 def test_doctype_decl_external(self): 157 inside = "DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN'" 158 self.check_events("<!%s>" % inside, [ 159 ("decl", inside), 160 ]) 161 162 def test_underscore_in_attrname(self): 163 # SF bug #436621 164 """Make sure attribute names with underscores are accepted""" 165 self.check_events("<a has_under _under>", [ 166 ("starttag", "a", [("has_under", "has_under"), 167 ("_under", "_under")]), 168 ]) 169 170 def test_underscore_in_tagname(self): 171 # SF bug #436621 172 """Make sure tag names with underscores are accepted""" 173 self.check_events("<has_under></has_under>", [ 174 ("starttag", "has_under", []), 175 ("endtag", "has_under"), 176 ]) 177 178 def test_quotes_in_unquoted_attrs(self): 179 # SF bug #436621 180 """Be sure quotes in unquoted attributes are made part of the value""" 181 self.check_events("<a href=foo'bar\"baz>", [ 182 ("starttag", "a", [("href", "foo'bar\"baz")]), 183 ]) 184 185 def test_xhtml_empty_tag(self): 186 """Handling of XHTML-style empty start tags""" 187 self.check_events("<br />text<i></i>", [ 188 ("starttag", "br", []), 189 ("data", "text"), 190 ("starttag", "i", []), 191 ("endtag", "i"), 192 ]) 193 194 def test_processing_instruction_only(self): 195 self.check_events("<?processing instruction>", [ 196 ("pi", "processing instruction"), 197 ]) 198 199 def test_bad_nesting(self): 200 self.check_events("<a><b></a></b>", [ 201 ("starttag", "a", []), 202 ("starttag", "b", []), 203 ("endtag", "a"), 204 ("endtag", "b"), 205 ]) 206 207 def test_bare_ampersands(self): 208 self.check_events("this text & contains & ampersands &", [ 209 ("data", "this text & contains & ampersands &"), 210 ]) 211 212 def test_bare_pointy_brackets(self): 213 self.check_events("this < text > contains < bare>pointy< brackets", [ 214 ("data", "this < text > contains < bare>pointy< brackets"), 215 ]) 216 217 def test_attr_syntax(self): 218 output = [ 219 ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", "e")]) 220 ] 221 self.check_events("""<a b='v' c="v" d=v e>""", output) 222 self.check_events("""<a b = 'v' c = "v" d = v e>""", output) 223 self.check_events("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output) 224 self.check_events("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output) 225 226 def test_attr_values(self): 227 self.check_events("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""", 228 [("starttag", "a", [("b", "xxx\n\txxx"), 229 ("c", "yyy\t\nyyy"), 230 ("d", "\txyz\n")]) 231 ]) 232 self.check_events("""<a b='' c="">""", [ 233 ("starttag", "a", [("b", ""), ("c", "")]), 234 ]) 235 # URL construction stuff from RFC 1808: 236 safe = "$-_.+" 237 extra = "!*'()," 238 reserved = ";/?:@&=" 239 url = "http://example.com:8080/path/to/file?%s%s%s" % ( 240 safe, extra, reserved) 241 self.check_events("""<e a=%s>""" % url, [ 242 ("starttag", "e", [("a", url)]), 243 ]) 244 # Regression test for SF patch #669683. 245 self.check_events("<e a=rgb(1,2,3)>", [ 246 ("starttag", "e", [("a", "rgb(1,2,3)")]), 247 ]) 248 249 def test_attr_values_entities(self): 250 """Substitution of entities and charrefs in attribute values""" 251 # SF bug #1452246 252 self.check_events("""<a b=< c=<> d=<-> e='< ' 253 f="&xxx;" g=' !' h='Ǵ' 254 i='x?a=b&c=d;' 255 j='&#42;' k='&#42;'>""", 256 [("starttag", "a", [("b", "<"), 257 ("c", "<>"), 258 ("d", "<->"), 259 ("e", "< "), 260 ("f", "&xxx;"), 261 ("g", " !"), 262 ("h", "Ǵ"), 263 ("i", "x?a=b&c=d;"), 264 ("j", "*"), 265 ("k", "*"), 266 ])]) 267 268 def test_convert_overrides(self): 269 # This checks that the character and entity reference 270 # conversion helpers are called at the documented times. No 271 # attempt is made to really change what the parser accepts. 272 # 273 self.collector = HTMLEntityCollector 274 self.check_events(('<a title="“test”">foo</a>' 275 '&foobar;*'), [ 276 ('entityref', 'convert', 'ldquo'), 277 ('charref', 'convert', 'x201d'), 278 ('starttag', 'a', [('title', '“test”')]), 279 ('data', 'foo'), 280 ('endtag', 'a'), 281 ('entityref', 'foobar'), 282 ('entityref', 'convert', 'foobar'), 283 ('charref', '42'), 284 ('charref', 'convert', '42'), 285 ('codepoint', 'convert', 42), 286 ]) 287 288 def test_attr_funky_names(self): 289 self.check_events("""<a a.b='v' c:d=v e-f=v>""", [ 290 ("starttag", "a", [("a.b", "v"), ("c:d", "v"), ("e-f", "v")]), 291 ]) 292 293 def test_attr_value_ip6_url(self): 294 # http://www.python.org/sf/853506 295 self.check_events(("<a href='http://[1080::8:800:200C:417A]/'>" 296 "<a href=http://[1080::8:800:200C:417A]/>"), [ 297 ("starttag", "a", [("href", "http://[1080::8:800:200C:417A]/")]), 298 ("starttag", "a", [("href", "http://[1080::8:800:200C:417A]/")]), 299 ]) 300 301 def test_weird_starttags(self): 302 self.check_events("<a<a>", [ 303 ("starttag", "a", []), 304 ("starttag", "a", []), 305 ]) 306 self.check_events("</a<a>", [ 307 ("endtag", "a"), 308 ("starttag", "a", []), 309 ]) 310 311 def test_declaration_junk_chars(self): 312 self.check_parse_error("<!DOCTYPE foo $ >") 313 314 def test_get_starttag_text(self): 315 s = """<foobar \n one="1"\ttwo=2 >""" 316 self.check_events(s, [ 317 ("starttag", "foobar", [("one", "1"), ("two", "2")]), 318 ]) 319 320 def test_cdata_content(self): 321 s = ("<cdata> <!-- not a comment --> ¬-an-entity-ref; </cdata>" 322 "<notcdata> <!-- comment --> </notcdata>") 323 self.collector = CDATAEventCollector 324 self.check_events(s, [ 325 ("starttag", "cdata", []), 326 ("data", " <!-- not a comment --> ¬-an-entity-ref; "), 327 ("endtag", "cdata"), 328 ("starttag", "notcdata", []), 329 ("data", " "), 330 ("comment", " comment "), 331 ("data", " "), 332 ("endtag", "notcdata"), 333 ]) 334 s = """<cdata> <not a='start tag'> </cdata>""" 335 self.check_events(s, [ 336 ("starttag", "cdata", []), 337 ("data", " <not a='start tag'> "), 338 ("endtag", "cdata"), 339 ]) 340 341 def test_illegal_declarations(self): 342 s = 'abc<!spacer type="block" height="25">def' 343 self.check_events(s, [ 344 ("data", "abc"), 345 ("unknown decl", 'spacer type="block" height="25"'), 346 ("data", "def"), 347 ]) 348 349 def test_enumerated_attr_type(self): 350 s = "<!DOCTYPE doc [<!ATTLIST doc attr (a | b) >]>" 351 self.check_events(s, [ 352 ('decl', 'DOCTYPE doc [<!ATTLIST doc attr (a | b) >]'), 353 ]) 354 355 def test_read_chunks(self): 356 # SF bug #1541697, this caused sgml parser to hang 357 # Just verify this code doesn't cause a hang. 358 CHUNK = 1024 # increasing this to 8212 makes the problem go away 359 360 f = open(test_support.findfile('sgml_input.html')) 361 fp = sgmllib.SGMLParser() 362 while 1: 363 data = f.read(CHUNK) 364 fp.feed(data) 365 if len(data) != CHUNK: 366 break 367 368 def test_only_decode_ascii(self): 369 # SF bug #1651995, make sure non-ascii character references are not decoded 370 s = '<signs exclamation="!" copyright="©" quoteleft="‘">' 371 self.check_events(s, [ 372 ('starttag', 'signs', 373 [('exclamation', '!'), ('copyright', '©'), 374 ('quoteleft', '‘')]), 375 ]) 376 377 # XXX These tests have been disabled by prefixing their names with 378 # an underscore. The first two exercise outstanding bugs in the 379 # sgmllib module, and the third exhibits questionable behavior 380 # that needs to be carefully considered before changing it. 381 382 def _test_starttag_end_boundary(self): 383 self.check_events("<a b='<'>", [("starttag", "a", [("b", "<")])]) 384 self.check_events("<a b='>'>", [("starttag", "a", [("b", ">")])]) 385 386 def _test_buffer_artefacts(self): 387 output = [("starttag", "a", [("b", "<")])] 388 self.check_events(["<a b='<'>"], output) 389 self.check_events(["<a ", "b='<'>"], output) 390 self.check_events(["<a b", "='<'>"], output) 391 self.check_events(["<a b=", "'<'>"], output) 392 self.check_events(["<a b='<", "'>"], output) 393 self.check_events(["<a b='<'", ">"], output) 394 395 output = [("starttag", "a", [("b", ">")])] 396 self.check_events(["<a b='>'>"], output) 397 self.check_events(["<a ", "b='>'>"], output) 398 self.check_events(["<a b", "='>'>"], output) 399 self.check_events(["<a b=", "'>'>"], output) 400 self.check_events(["<a b='>", "'>"], output) 401 self.check_events(["<a b='>'", ">"], output) 402 403 output = [("comment", "abc")] 404 self.check_events(["", "<!--abc-->"], output) 405 self.check_events(["<", "!--abc-->"], output) 406 self.check_events(["<!", "--abc-->"], output) 407 self.check_events(["<!-", "-abc-->"], output) 408 self.check_events(["<!--", "abc-->"], output) 409 self.check_events(["<!--a", "bc-->"], output) 410 self.check_events(["<!--ab", "c-->"], output) 411 self.check_events(["<!--abc", "-->"], output) 412 self.check_events(["<!--abc-", "->"], output) 413 self.check_events(["<!--abc--", ">"], output) 414 self.check_events(["<!--abc-->", ""], output) 415 416 def _test_starttag_junk_chars(self): 417 self.check_parse_error("<") 418 self.check_parse_error("<>") 419 self.check_parse_error("</$>") 420 self.check_parse_error("</") 421 self.check_parse_error("</a") 422 self.check_parse_error("<$") 423 self.check_parse_error("<$>") 424 self.check_parse_error("<!") 425 self.check_parse_error("<a $>") 426 self.check_parse_error("<a") 427 self.check_parse_error("<a foo='bar'") 428 self.check_parse_error("<a foo='bar") 429 self.check_parse_error("<a foo='>'") 430 self.check_parse_error("<a foo='>") 431 self.check_parse_error("<a foo=>") 432 433 434 def test_main(): 435 test_support.run_unittest(SGMLParserTestCase) 436 437 438 if __name__ == "__main__": 439 test_main() 440
