1 #!/usr/bin/env python3 2 """ 3 Utility for parsing HTML5 entity definitions available from: 4 5 http://dev.w3.org/html5/spec/entities.json 6 7 Written by Ezio Melotti and Iuliia Proskurnia. 8 9 """ 10 11 import os 12 import sys 13 import json 14 from urllib.request import urlopen 15 from html.entities import html5 16 17 entities_url = 'http://dev.w3.org/html5/spec/entities.json' 18 19 def get_json(url): 20 """Download the json file from the url and returns a decoded object.""" 21 with urlopen(url) as f: 22 data = f.read().decode('utf-8') 23 return json.loads(data) 24 25 def create_dict(entities): 26 """Create the html5 dict from the decoded json object.""" 27 new_html5 = {} 28 for name, value in entities.items(): 29 new_html5[name.lstrip('&')] = value['characters'] 30 return new_html5 31 32 def compare_dicts(old, new): 33 """Compare the old and new dicts and print the differences.""" 34 added = new.keys() - old.keys() 35 if added: 36 print('{} entitie(s) have been added:'.format(len(added))) 37 for name in sorted(added): 38 print(' {!r}: {!r}'.format(name, new[name])) 39 removed = old.keys() - new.keys() 40 if removed: 41 print('{} entitie(s) have been removed:'.format(len(removed))) 42 for name in sorted(removed): 43 print(' {!r}: {!r}'.format(name, old[name])) 44 changed = set() 45 for name in (old.keys() & new.keys()): 46 if old[name] != new[name]: 47 changed.add((name, old[name], new[name])) 48 if changed: 49 print('{} entitie(s) have been modified:'.format(len(changed))) 50 for item in sorted(changed): 51 print(' {!r}: {!r} -> {!r}'.format(*item)) 52 53 def write_items(entities, file=sys.stdout): 54 """Write the items of the dictionary in the specified file.""" 55 # The keys in the generated dictionary should be sorted 56 # in a case-insensitive way, however, when two keys are equal, 57 # the uppercase version should come first so that the result 58 # looks like: ['Aacute', 'aacute', 'Aacute;', 'aacute;', ...] 59 # To do this we first sort in a case-sensitive way (so all the 60 # uppercase chars come first) and then sort with key=str.lower. 61 # Since the sorting is stable the uppercase keys will eventually 62 # be before their equivalent lowercase version. 63 keys = sorted(entities.keys()) 64 keys = sorted(keys, key=str.lower) 65 print('html5 = {', file=file) 66 for name in keys: 67 print(' {!r}: {!a},'.format(name, entities[name]), file=file) 68 print('}', file=file) 69 70 71 if __name__ == '__main__': 72 # without args print a diff between html.entities.html5 and new_html5 73 # with --create print the new html5 dict 74 # with --patch patch the Lib/html/entities.py file 75 new_html5 = create_dict(get_json(entities_url)) 76 if '--create' in sys.argv: 77 print('# map the HTML5 named character references to the ' 78 'equivalent Unicode character(s)') 79 print('# Generated by {}. Do not edit manually.'.format(__file__)) 80 write_items(new_html5) 81 elif '--patch' in sys.argv: 82 fname = 'Lib/html/entities.py' 83 temp_fname = fname + '.temp' 84 with open(fname) as f1, open(temp_fname, 'w') as f2: 85 skip = False 86 for line in f1: 87 if line.startswith('html5 = {'): 88 write_items(new_html5, file=f2) 89 skip = True 90 continue 91 if skip: 92 # skip the old items until the } 93 if line.startswith('}'): 94 skip = False 95 continue 96 f2.write(line) 97 os.remove(fname) 98 os.rename(temp_fname, fname) 99 else: 100 if html5 == new_html5: 101 print('The current dictionary is updated.') 102 else: 103 compare_dicts(html5, new_html5) 104 print('Run "./python {0} --patch" to update Lib/html/entities.html ' 105 'or "./python {0} --create" to see the generated ' 'dictionary.'.format(__file__)) 106
