1 #!/usr/bin/env python3 2 # This script converts a C file to use the PEP 384 type definition API 3 # Usage: abitype.py < old_code > new_code 4 import re, sys 5 6 ###### Replacement of PyTypeObject static instances ############## 7 8 # classify each token, giving it a one-letter code: 9 # S: static 10 # T: PyTypeObject 11 # I: ident 12 # W: whitespace 13 # =, {, }, ; : themselves 14 def classify(): 15 res = [] 16 for t,v in tokens: 17 if t == 'other' and v in "={};": 18 res.append(v) 19 elif t == 'ident': 20 if v == 'PyTypeObject': 21 res.append('T') 22 elif v == 'static': 23 res.append('S') 24 else: 25 res.append('I') 26 elif t == 'ws': 27 res.append('W') 28 else: 29 res.append('.') 30 return ''.join(res) 31 32 # Obtain a list of fields of a PyTypeObject, in declaration order, 33 # skipping ob_base 34 # All comments are dropped from the variable (which are typically 35 # just the slot names, anyway), and information is discarded whether 36 # the original type was static. 37 def get_fields(start, real_end): 38 pos = start 39 # static? 40 if tokens[pos][1] == 'static': 41 pos += 2 42 # PyTypeObject 43 pos += 2 44 # name 45 name = tokens[pos][1] 46 pos += 1 47 while tokens[pos][1] != '{': 48 pos += 1 49 pos += 1 50 # PyVarObject_HEAD_INIT 51 while tokens[pos][0] in ('ws', 'comment'): 52 pos += 1 53 if tokens[pos][1] != 'PyVarObject_HEAD_INIT': 54 raise Exception('%s has no PyVarObject_HEAD_INIT' % name) 55 while tokens[pos][1] != ')': 56 pos += 1 57 pos += 1 58 # field definitions: various tokens, comma-separated 59 fields = [] 60 while True: 61 while tokens[pos][0] in ('ws', 'comment'): 62 pos += 1 63 end = pos 64 while tokens[end][1] not in ',}': 65 if tokens[end][1] == '(': 66 nesting = 1 67 while nesting: 68 end += 1 69 if tokens[end][1] == '(': nesting+=1 70 if tokens[end][1] == ')': nesting-=1 71 end += 1 72 assert end < real_end 73 # join field, excluding separator and trailing ws 74 end1 = end-1 75 while tokens[end1][0] in ('ws', 'comment'): 76 end1 -= 1 77 fields.append(''.join(t[1] for t in tokens[pos:end1+1])) 78 if tokens[end][1] == '}': 79 break 80 pos = end+1 81 return name, fields 82 83 # List of type slots as of Python 3.2, omitting ob_base 84 typeslots = [ 85 'tp_name', 86 'tp_basicsize', 87 'tp_itemsize', 88 'tp_dealloc', 89 'tp_print', 90 'tp_getattr', 91 'tp_setattr', 92 'tp_reserved', 93 'tp_repr', 94 'tp_as_number', 95 'tp_as_sequence', 96 'tp_as_mapping', 97 'tp_hash', 98 'tp_call', 99 'tp_str', 100 'tp_getattro', 101 'tp_setattro', 102 'tp_as_buffer', 103 'tp_flags', 104 'tp_doc', 105 'tp_traverse', 106 'tp_clear', 107 'tp_richcompare', 108 'tp_weaklistoffset', 109 'tp_iter', 110 'iternextfunc', 111 'tp_methods', 112 'tp_members', 113 'tp_getset', 114 'tp_base', 115 'tp_dict', 116 'tp_descr_get', 117 'tp_descr_set', 118 'tp_dictoffset', 119 'tp_init', 120 'tp_alloc', 121 'tp_new', 122 'tp_free', 123 'tp_is_gc', 124 'tp_bases', 125 'tp_mro', 126 'tp_cache', 127 'tp_subclasses', 128 'tp_weaklist', 129 'tp_del', 130 'tp_version_tag', 131 ] 132 133 # Generate a PyType_Spec definition 134 def make_slots(name, fields): 135 res = [] 136 res.append('static PyType_Slot %s_slots[] = {' % name) 137 # defaults for spec 138 spec = { 'tp_itemsize':'0' } 139 for i, val in enumerate(fields): 140 if val.endswith('0'): 141 continue 142 if typeslots[i] in ('tp_name', 'tp_doc', 'tp_basicsize', 143 'tp_itemsize', 'tp_flags'): 144 spec[typeslots[i]] = val 145 continue 146 res.append(' {Py_%s, %s},' % (typeslots[i], val)) 147 res.append('};') 148 res.append('static PyType_Spec %s_spec = {' % name) 149 res.append(' %s,' % spec['tp_name']) 150 res.append(' %s,' % spec['tp_basicsize']) 151 res.append(' %s,' % spec['tp_itemsize']) 152 res.append(' %s,' % spec['tp_flags']) 153 res.append(' %s_slots,' % name) 154 res.append('};\n') 155 return '\n'.join(res) 156 157 158 if __name__ == '__main__': 159 160 ############ Simplistic C scanner ################################## 161 tokenizer = re.compile( 162 r"(?P<preproc>#.*\n)" 163 r"|(?P<comment>/\*.*?\*/)" 164 r"|(?P<ident>[a-zA-Z_][a-zA-Z0-9_]*)" 165 r"|(?P<ws>[ \t\n]+)" 166 r"|(?P<other>.)", 167 re.MULTILINE) 168 169 tokens = [] 170 source = sys.stdin.read() 171 pos = 0 172 while pos != len(source): 173 m = tokenizer.match(source, pos) 174 tokens.append([m.lastgroup, m.group()]) 175 pos += len(tokens[-1][1]) 176 if tokens[-1][0] == 'preproc': 177 # continuation lines are considered 178 # only in preprocess statements 179 while tokens[-1][1].endswith('\\\n'): 180 nl = source.find('\n', pos) 181 if nl == -1: 182 line = source[pos:] 183 else: 184 line = source[pos:nl+1] 185 tokens[-1][1] += line 186 pos += len(line) 187 188 # Main loop: replace all static PyTypeObjects until 189 # there are none left. 190 while 1: 191 c = classify() 192 m = re.search('(SW)?TWIW?=W?{.*?};', c) 193 if not m: 194 break 195 start = m.start() 196 end = m.end() 197 name, fields = get_fields(start, end) 198 tokens[start:end] = [('',make_slots(name, fields))] 199 200 # Output result to stdout 201 for t, v in tokens: 202 sys.stdout.write(v) 203
