1 # Copyright 2016, VIXL authors 2 # All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are met: 6 # 7 # * Redistributions of source code must retain the above copyright notice, 8 # this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above copyright notice, 10 # this list of conditions and the following disclaimer in the documentation 11 # and/or other materials provided with the distribution. 12 # * Neither the name of ARM Limited nor the names of its contributors may be 13 # used to endorse or promote products derived from this software without 14 # specific prior written permission. 15 # 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 17 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 20 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 27 import json 28 import re 29 import os 30 import hashlib 31 import collections 32 import itertools 33 34 from test_generator import data_types 35 from test_generator import generator 36 37 class DataTypeBuilder(object): 38 """ 39 Factory object for building `data_types.Operand` and `data_types.Input` 40 objects. This object stores information about all operand and input types 41 described in JSON as dictionnaries indexed by their identifier. See 42 `test/a32/config/data-types.json` as a reference. 43 44 Attributes: 45 operand_types Dictionnary of type names corresponding to the JSON 46 "type" field. 47 operand_variants Dictionnary of (variants, default) tuples. 48 49 input_types Dictionnary of type names corresponding to the JSON 50 "type" field. 51 input_values Dictionnary of (values, default) tuples. 52 """ 53 54 def __init__(self, operand_types, operand_variants, input_types, 55 input_values): 56 self.operand_types = operand_types 57 self.operand_variants = operand_variants 58 self.input_types = input_types 59 self.input_values = input_values 60 61 def BuildOperand(self, name, identifier): 62 """ 63 Build a `data_types.Operand` object with the name `name`. `identifier` 64 identifies which type we want to create, as declared in JSON. 65 """ 66 type_name = self.operand_types[identifier] 67 variants, default = self.operand_variants[identifier] 68 # We simply pass the `type_name` as a parameter which will be used verbatim 69 # in the code. 70 return data_types.Operand(name, type_name, variants, default) 71 72 def BuildInput(self, name, identifier): 73 """ 74 Build a `data_types.Input` object with the name `name`. `identifier` 75 identifies which type we want to create, as declared in JSON. 76 """ 77 type_name = self.input_types[identifier] 78 values, default = self.input_values[identifier] 79 # For `data_types.Input` types, the `type_name` refers to the actual name of 80 # the Python class, inheriting from `Input`. This is done so that different 81 # input types can generate different C++ code by overriding the `Load` and 82 # `Store` methods. 83 input_constructor = getattr(data_types, type_name) 84 return input_constructor(name, values, default) 85 86 87 def LoadJSON(filename): 88 """ 89 Read `filename`, strip its comments and load as JSON. 90 """ 91 with open(filename, "r") as f: 92 match_cpp_comments = re.compile("//.*\n") 93 # The order in which structures are described in JSON matters as we use them 94 # as a seed. Computing a hash from a unordered dict always gives a different 95 # value. We use the `object_pairs_hook` to make the json module create 96 # `OrderedDict` objects instead of builtin `dict` objects. 97 return json.loads(match_cpp_comments.sub("", f.read()), 98 object_pairs_hook=collections.OrderedDict) 99 100 101 def ParseDataTypes(json_data_types): 102 """ 103 Build a `DataTypeBuilder` object containing all information from the JSON 104 description in `json_data_types`. 105 106 ~~~ 107 { 108 "operands": [ 109 { 110 "identifier": "AllRegistersButPC" 111 "type": "Register" 112 "variants": [ 113 "r0", 114 "r1", 115 "r2", 116 "r3" 117 ] 118 "default": "r0" 119 }, 120 { 121 ... 122 } 123 ], 124 "inputs": [ 125 { 126 "identifier": "Register" 127 "type": "Register" 128 "values": [ 129 "0x00000000", 130 "0xffffffff", 131 "0xabababab" 132 ] 133 "default": "0xabababab" 134 }, 135 { 136 ... 137 } 138 ] 139 } 140 ~~~ 141 """ 142 operand_types = { 143 json_operand_type["identifier"]: json_operand_type["type"] 144 for json_operand_type in json_data_types["operands"] 145 } 146 operand_variants = { 147 json_operand_type["identifier"]: 148 (json_operand_type["variants"], json_operand_type["default"]) 149 for json_operand_type in json_data_types["operands"] 150 } 151 input_types = { 152 json_input_type["identifier"]: json_input_type["type"] 153 for json_input_type in json_data_types["inputs"] 154 } 155 input_values = { 156 json_input_type["identifier"]: 157 (json_input_type["values"], json_input_type["default"]) 158 for json_input_type in json_data_types["inputs"] 159 } 160 return DataTypeBuilder(operand_types, operand_variants, input_types, input_values) 161 162 163 def ParseDescription(data_type_builder, json_description): 164 """ 165 Parse the instruction description into a 166 (`generator.OperandList`, `generator.InputList`) tuple. 167 168 Example for an instruction that takes a condidition code, two registers and an 169 immediate as operand. It will also need inputs for the registers, as well as 170 NZCV flags. 171 ~~~ 172 { 173 "operands": [ 174 { 175 "name": "cond", 176 "type": "Condition", 177 }, 178 { 179 "name": "rd", 180 "type": "RegisterScratch", 181 }, 182 { 183 "name": "rn", 184 "type": "RegisterScratch", 185 }, 186 // The last operand needs to be wrapped into a C++ `Operand` object. We 187 // declare the operands that need to be wrapped as a list. 188 { 189 "name": "op", 190 "wrapper": "Operand", 191 "operands": [ 192 { 193 "name": "immediate", 194 "type": "ModifiedImmediate", 195 } 196 ] 197 } 198 ], 199 "inputs": [ 200 { 201 "name": "apsr", 202 "type": "NZCV" 203 }, 204 { 205 "name": "rd", 206 "type": "Register" 207 }, 208 { 209 "name": "rn", 210 "type": "Register" 211 } 212 ] 213 ] 214 ~~~ 215 """ 216 217 operands = [] 218 for json_operand in json_description["operands"]: 219 if "name" in json_operand and "type" in json_operand: 220 operands.append(data_type_builder.BuildOperand(json_operand["name"], 221 json_operand["type"])) 222 elif "name" in json_operand and \ 223 "wrapper" in json_operand and \ 224 "operands" in json_operand: 225 wrapped_operands = [ 226 data_type_builder.BuildOperand(json_wrapped_operand["name"], 227 json_wrapped_operand["type"]) 228 for json_wrapped_operand in json_operand["operands"] 229 ] 230 operands.append(data_types.OperandWrapper(json_operand["name"], 231 json_operand["wrapper"], 232 wrapped_operands)) 233 else: 234 raise Exception("Parser failed to recognize JSON \"description\".") 235 operand_list = generator.OperandList(operands) 236 237 json_description_inputs = json_description["inputs"] 238 input_list = generator.InputList([ 239 data_type_builder.BuildInput(json_input["name"], json_input["type"]) 240 for json_input in json_description_inputs 241 ]) 242 243 return operand_list, input_list 244 245 246 def ParseTestCase(json_test_case): 247 """ 248 Build a `generator.TestCase` object from its JSON description. 249 250 ~~~ 251 { 252 "name": "RdIsNotRn", 253 "operands": [ 254 "rd", "rn" 255 ], 256 "inputs": [ 257 "rd", "rn" 258 ], 259 "operand-filter": "rd != rn", // Python code to limit operand generation. 260 "operand-limit": 10 // Choose a random sample of 10 operands. 261 } 262 ... 263 { 264 "name": "Flags", 265 "operands": [ 266 "cond" 267 ], 268 "inputs": [ 269 "apsr", "q" 270 ], 271 "input-filter": "q == \"QFlag\"", // Python code to limit input generation 272 "input-limit": 200 // Choose a random sample of 200 inputs. 273 } 274 ... 275 { 276 "name": "InITBlock", 277 "operands": [ 278 "cond", "rd", "rn", "rm" 279 ], 280 "in-it-block": "{cond}", // Generate an extra IT instruction. This string 281 // will be used as the operand passed to IT. One 282 // needs to specify under what name the condition 283 // operand is represented, in braces. 284 "operand-filter": "cond != 'al' and rd == rm" 285 } 286 ~~~ 287 """ 288 289 # TODO: The fields in "operands" and "inputs" respectively refer to operands 290 # and inputs declared in the instruction description (see `ParseDescription`). 291 # We should assert that the user hasn't miss typed them and raise an 292 # exception. 293 294 # If the fields are not present, give them default values (empty list, 295 # "True", or "None"). 296 operand_names = json_test_case["operands"] \ 297 if "operands" in json_test_case else [] 298 input_names = json_test_case["inputs"] if "inputs" in json_test_case else [] 299 operand_filter = json_test_case["operand-filter"] \ 300 if "operand-filter" in json_test_case else "True" 301 input_filter = json_test_case["input-filter"] \ 302 if "input-filter" in json_test_case else "True" 303 operand_limit = json_test_case["operand-limit"] \ 304 if "operand-limit" in json_test_case else None 305 input_limit = json_test_case["input-limit"] \ 306 if "input-limit" in json_test_case else None 307 in_it_block = json_test_case["in-it-block"] \ 308 if "in-it-block" in json_test_case else None 309 310 # Create a seed from the test case description. It will only change if the 311 # test case has changed. 312 md5 = hashlib.md5(str(json_test_case).encode()) 313 seed = md5.hexdigest() 314 315 return generator.TestCase(json_test_case["name"], seed, operand_names, input_names, 316 operand_filter, input_filter, operand_limit, 317 input_limit, in_it_block) 318 319 320 def ParseTestFile(test_name, test_isa, mnemonics, operand_list, input_list, 321 json_test_file): 322 """ 323 Build a `generator.Generator` object from a test file description. We have one 324 for each generated test files. 325 326 ~~~ 327 { 328 "type": "simulator", // Type of the test. This will control the prefix we 329 // use when naming the file to generate. 330 "name": "special-case", // Optional name that will be included in the 331 // generated filename. 332 "mnemonics": [ // Optional list of instruction, overriding the top-level 333 "Adc", // one. 334 "Add", 335 ... 336 ], 337 "test-cases": [ 338 ... // Test case descriptions parsed with `ParseTestCase`. 339 ] 340 } 341 ~~~ 342 """ 343 name = json_test_file["name"] if "name" in json_test_file else "" 344 if name is not "": 345 test_name = test_name + "-" + name 346 # Override the top-level mnemonics list with a subset. 347 if "mnemonics" in json_test_file: 348 if set(json_test_file["mnemonics"]) == set(mnemonics): 349 raise Exception( 350 "Overriding mnemonic list is identical to the top-level list") 351 if not(set(json_test_file["mnemonics"]) < set(mnemonics)): 352 raise Exception( 353 "Overriding mnemonic list should a subset of the top-level list") 354 mnemonics = json_test_file["mnemonics"] 355 test_cases = [ 356 ParseTestCase(json_test_case) 357 for json_test_case in json_test_file["test-cases"] 358 ] 359 return generator.Generator(test_name, test_isa, json_test_file["type"], 360 mnemonics, operand_list, input_list, test_cases) 361 362 363 def ParseConfig(test_name, test_isas, data_type_builder, json_config): 364 """ 365 Return a list of `generator.Generator` objects from a JSON description. This 366 is the top-level description. 367 368 ~~~ 369 { 370 "mnemonics": [ 371 "Adc", 372 "Add", 373 ... 374 ], 375 "description": [ 376 ... // Instruction description parsed with `ParseDescription`. 377 ], 378 "test-files": [ 379 ... // Test files descriptions parsed with `ParseTestFile`. 380 ] 381 } 382 ~~~ 383 """ 384 mnemonics = json_config["mnemonics"] 385 operand_list, input_list = ParseDescription( 386 data_type_builder, json_config["description"]) 387 388 return itertools.chain(*[[ 389 ParseTestFile(test_name, test_isa, mnemonics, operand_list, 390 input_list, json_test_file) 391 for json_test_file in json_config["test-files"] 392 ] 393 for test_isa in test_isas 394 ]) 395 396 397 def GetTestNameAndISAFromFileName(filename): 398 """ 399 Return a tuple (name, [isa, ...]) extracted from the file name. 400 """ 401 # Strip the ".json" extension 402 stripped_basename = os.path.splitext(os.path.basename(filename))[0] 403 # The ISA is the last element in the filename, seperated with "-". 404 if stripped_basename.endswith(('-a32', '-t32')): 405 isa = [stripped_basename[-3:]] 406 test_name = stripped_basename[:-4] 407 else: 408 # If the ISA is ommitted, support both. 409 isa = ["a32", "t32"] 410 test_name = stripped_basename 411 412 return (test_name, isa) 413 414 415 def GetTestNameFromFileName(filename): 416 """ 417 Return the name given to this test from its file name, stripped of the 418 optional "a32" or "t32" at the end. 419 """ 420 test_name, _ = GetTestNameAndISAFromFileName(filename) 421 return test_name 422 423 424 def GetISAsFromFileName(filename): 425 """ 426 Return a list of ISAs supported by the test, from the file name, either 427 ["a32"], ["t32"] or both. 428 """ 429 _, isas = GetTestNameAndISAFromFileName(filename) 430 431 return isas 432 433 def Parse(data_type_file, config_files): 434 """ 435 Parse the `data_type_file` and `test_case_files` json description files into a 436 list of (name, test_case) tuples. Test cases are `generator.TestCase` 437 objects that can be used to generate C++. 438 """ 439 440 # Create a `DataTypeBuilder` object. This object will passed down and used to 441 # instantiate `data_types.Operand` and `data_types.Input` objects. 442 data_type_builder = ParseDataTypes(LoadJSON(data_type_file)) 443 444 # Build a list of (name, JSON) tuples to represent the new tests. 445 json_configs = [ 446 # We create the name of the test by looking at the file name stripped of 447 # its extension. 448 (GetTestNameFromFileName(config_file), GetISAsFromFileName(config_file), 449 LoadJSON(config_file)) 450 for config_file in config_files 451 ] 452 453 # Return a list of Generator objects. The generator is the entry point to 454 # generate a file. 455 # Note that `ParseConfig` returns a list of generators already. We use `chain` 456 # here to flatten a list of lists into just a list. 457 return itertools.chain(*[ 458 ParseConfig(test_name, test_isas, data_type_builder, json_config) 459 for test_name, test_isas, json_config in json_configs 460 ]) 461