1 #!/usr/bin/python 2 3 """A script that provides convertion between models.job and a protocol 4 buffer object. 5 6 This script contains only one class that takes an job instance and 7 convert it into a protocol buffer object. The class will also be 8 responsible for serializing the job instance via protocol buffers. 9 10 """ 11 12 # import python libraries 13 import datetime 14 import time 15 import logging 16 17 # import autotest libraries 18 from autotest_lib.tko import models 19 from autotest_lib.tko import tko_pb2 20 from autotest_lib.tko import utils 21 22 __author__ = 'darrenkuo (at] google.com (Darren Kuo)' 23 24 mktime = time.mktime 25 datetime = datetime.datetime 26 27 class JobSerializer(object): 28 """A class that takes a job object of the tko module and package 29 it with a protocol buffer. 30 31 This class will take a model.job object as input and create a 32 protocol buffer to include all the content of the job object. This 33 protocol buffer object will be serialized into a binary file. 34 """ 35 36 def __init__(self): 37 38 self.job_type_dict = {'dir':str, 'tests':list, 'user':str, 39 'label':str, 'machine':str, 40 'queued_time':datetime, 41 'started_time':datetime, 42 'finished_time':datetime, 43 'machine_owner':str, 44 'machine_group':str, 'aborted_by':str, 45 'aborted_on':datetime, 46 'keyval_dict':dict, 47 'afe_parent_job_id':str, 48 'build_version':str, 49 'suite':str, 50 'board':str} 51 52 self.test_type_dict = {'subdir':str, 'testname':str, 53 'status':str, 'reason':str, 54 'kernel':models.kernel, 'machine':str, 55 'started_time':datetime, 56 'finished_time':datetime, 57 'iterations':list, 'attributes':dict, 58 'labels':list} 59 60 self.kernel_type_dict = {'base':str, 'kernel_hash':str} 61 62 self.iteration_type_dict = {'index':int, 'attr_keyval':dict, 63 'perf_keyval':dict} 64 65 66 def deserialize_from_binary(self, infile): 67 """Takes in a binary file name and returns a tko job object. 68 69 The method first deserialize the binary into a protocol buffer 70 job object and then converts the job object into a tko job 71 object. 72 73 74 @param infile: the name of the binary file that will be deserialized. 75 76 @return: a tko job that is represented by the binary file will 77 be returned. 78 """ 79 80 job_pb = tko_pb2.Job() 81 82 binary = open(infile, 'r') 83 try: 84 job_pb.ParseFromString(binary.read()) 85 finally: 86 binary.close() 87 88 return self.get_tko_job(job_pb) 89 90 91 def serialize_to_binary(self, the_job, tag, binaryfilename): 92 """Serializes the tko job object into a binary by using a 93 protocol buffer. 94 95 The method takes a tko job object and constructs a protocol 96 buffer job object. Then invokes the native serializing 97 function on the object to get a binary string. The string is 98 then written to outfile. 99 100 Precondition: Assumes that all the information about the job 101 is already in the job object. Any fields that is None will be 102 provided a default value. 103 104 @param the_job: the tko job object that will be serialized. 105 tag: contains the job name and the afe_job_id 106 binaryfilename: the name of the file that will be written to 107 @param tag: The job tag string. 108 @param binaryfilename: The output filename. 109 110 @return: the filename of the file that contains the 111 binary of the serialized object. 112 """ 113 114 pb_job = tko_pb2.Job() 115 self.set_pb_job(the_job, pb_job, tag) 116 117 out = open(binaryfilename, 'wb') 118 try: 119 out.write(pb_job.SerializeToString()) 120 finally: 121 out.close() 122 123 124 def set_afe_job_id_and_tag(self, pb_job, tag): 125 """Sets the pb job's afe_job_id and tag field. 126 127 @param 128 pb_job: the pb job that will have it's fields set. 129 tag: used to set pb_job.tag and pb_job.afe_job_id. 130 """ 131 pb_job.tag = tag 132 pb_job.afe_job_id = utils.get_afe_job_id(tag) 133 134 135 # getter setter methods 136 def get_tko_job(self, job): 137 """Creates a a new tko job object from the pb job object. 138 139 Uses getter methods on the pb objects to extract all the 140 attributes and finally constructs a tko job object using the 141 models.job constructor. 142 143 @param 144 job: a pb job where data is being extracted from. 145 146 @return a tko job object. 147 """ 148 149 fields_dict = self.get_trivial_attr(job, self.job_type_dict) 150 151 fields_dict['tests'] = [self.get_tko_test(test) for test in job.tests] 152 153 fields_dict['keyval_dict'] = dict((keyval.name, keyval.value) 154 for keyval in job.keyval_dict) 155 156 newjob = models.job(fields_dict['dir'], fields_dict['user'], 157 fields_dict['label'], 158 fields_dict['machine'], 159 fields_dict['queued_time'], 160 fields_dict['started_time'], 161 fields_dict['finished_time'], 162 fields_dict['machine_owner'], 163 fields_dict['machine_group'], 164 fields_dict['aborted_by'], 165 fields_dict['aborted_on'], 166 fields_dict['keyval_dict']) 167 168 newjob.tests.extend(fields_dict['tests']) 169 170 return newjob 171 172 173 def set_pb_job(self, tko_job, pb_job, tag): 174 """Set the fields for the new job object. 175 176 Method takes in a tko job and an empty protocol buffer job 177 object. Then safely sets all the appropriate field by first 178 testing if the value in the original object is None. 179 180 @param 181 tko_job: a tko job instance that will have it's values 182 transfered to the new job 183 pb_job: a new instance of the job class provided in the 184 protocol buffer. 185 tag: used to set pb_job.tag and pb_job.afe_job_id. 186 """ 187 188 self.set_trivial_attr(tko_job, pb_job, self.job_type_dict) 189 self.set_afe_job_id_and_tag(pb_job, tag) 190 if hasattr(tko_job, 'index'): 191 pb_job.job_idx = tko_job.index 192 193 for test in tko_job.tests: 194 newtest = pb_job.tests.add() 195 self.set_pb_test(test, newtest) 196 197 for key, val in tko_job.keyval_dict.iteritems(): 198 newkeyval = pb_job.keyval_dict.add() 199 newkeyval.name = key 200 newkeyval.value = str(val) 201 202 203 def get_tko_test(self, test): 204 """Creates a tko test from pb_test. 205 206 Extracts data from pb_test by calling helper methods and 207 creates a tko test using the models.test constructor. 208 209 @param: 210 test: a pb_test where fields will be extracted from. 211 212 @return a new instance of models.test 213 """ 214 fields_dict = self.get_trivial_attr(test, self.test_type_dict) 215 216 fields_dict['kernel'] = self.get_tko_kernel(test.kernel) 217 218 fields_dict['iterations'] = [self.get_tko_iteration(iteration) 219 for iteration in test.iterations] 220 221 fields_dict['attributes'] = dict((keyval.name, keyval.value) 222 for keyval in test.attributes) 223 224 fields_dict['labels'] = list(test.labels) 225 226 # The constructor for models.test accepts a "perf_values" list that 227 # represents performance values of the test. The empty list argument 228 # in the constructor call below represents this value and makes this 229 # code adhere properly to the models.test constructor argument list. 230 # However, the effect of the empty list is that perf values are 231 # ignored in the job_serializer module. This is ok for now because 232 # autotest does not use the current module. If job_serializer is used 233 # in the future, we need to modify the "tko.proto" protobuf file to 234 # understand the notion of perf_values, then modify this file 235 # accordingly to use it. 236 return models.test(fields_dict['subdir'], 237 fields_dict['testname'], 238 fields_dict['status'], 239 fields_dict['reason'], 240 fields_dict['kernel'], 241 fields_dict['machine'], 242 fields_dict['started_time'], 243 fields_dict['finished_time'], 244 fields_dict['iterations'], 245 fields_dict['attributes'], 246 [], 247 fields_dict['labels']) 248 249 250 def set_pb_test(self, tko_test, pb_test): 251 """Sets the various fields of test object of the tko protocol. 252 253 Method takes a tko test and a new test of the protocol buffer and 254 transfers the values in the tko test to the new test. 255 256 @param 257 tko_test: a tko test instance. 258 pb_test: an empty protocol buffer test instance. 259 260 """ 261 262 self.set_trivial_attr(tko_test, pb_test, self.test_type_dict) 263 264 self.set_pb_kernel(tko_test.kernel, pb_test.kernel) 265 if hasattr(tko_test, 'test_idx'): 266 pb_test.test_idx = tko_test.test_idx 267 268 for current_iteration in tko_test.iterations: 269 pb_iteration = pb_test.iterations.add() 270 self.set_pb_iteration(current_iteration, pb_iteration) 271 272 for key, val in tko_test.attributes.iteritems(): 273 newkeyval = pb_test.attributes.add() 274 newkeyval.name = key 275 newkeyval.value = str(val) 276 277 for current_label in tko_test.labels: 278 pb_test.labels.append(current_label) 279 280 281 def get_tko_kernel(self, kernel): 282 """Constructs a new tko kernel object from a pb kernel object. 283 284 Uses all the getter methods on the pb kernel object to extract 285 the attributes and constructs a new tko kernel object using 286 the model.kernel constructor. 287 288 @param 289 kernel: a pb kernel object where data will be extracted. 290 291 @return a new tko kernel object. 292 """ 293 294 fields_dict = self.get_trivial_attr(kernel, self.kernel_type_dict) 295 296 return models.kernel(fields_dict['base'], [], fields_dict['kernel_hash']) 297 298 299 def set_pb_kernel(self, tko_kernel, pb_kernel): 300 """Set a specific kernel of a test. 301 302 Takes the same form of all the other setting methods. It 303 seperates the string variables from the int variables and set 304 them safely. 305 306 @param 307 tko_kernel: a tko kernel. 308 pb_kernel: an empty protocol buffer kernel. 309 310 """ 311 312 self.set_trivial_attr(tko_kernel, pb_kernel, self.kernel_type_dict) 313 314 315 def get_tko_iteration(self, iteration): 316 """Creates a new tko iteration with the data in the provided 317 pb iteration. 318 319 Uses the data in the pb iteration and the models.iteration 320 constructor to create a new tko iterations 321 322 @param 323 iteration: a pb iteration instance 324 325 @return a tko iteration instance with the same data. 326 """ 327 328 fields_dict = self.get_trivial_attr(iteration, 329 self.iteration_type_dict) 330 331 fields_dict['attr_keyval'] = dict((keyval.name, keyval.value) 332 for keyval in iteration.attr_keyval) 333 334 fields_dict['perf_keyval'] = dict((keyval.name, keyval.value) 335 for keyval in iteration.perf_keyval) 336 337 return models.iteration(fields_dict['index'], 338 fields_dict['attr_keyval'], 339 fields_dict['perf_keyval']) 340 341 342 def set_pb_iteration(self, tko_iteration, pb_iteration): 343 """Sets all fields for a particular iteration. 344 345 Takes same form as all the other setting methods. Sets int, 346 str and datetime variables safely. 347 348 @param 349 tko_iteration: a tko test iteration. 350 pb_iteration: an empty pb test iteration. 351 352 """ 353 354 self.set_trivial_attr(tko_iteration, pb_iteration, 355 self.iteration_type_dict) 356 357 for key, val in tko_iteration.attr_keyval.iteritems(): 358 newkeyval = pb_iteration.attr_keyval.add() 359 newkeyval.name = key 360 newkeyval.value = str(val) 361 362 for key, val in tko_iteration.perf_keyval.iteritems(): 363 newkeyval = pb_iteration.perf_keyval.add() 364 newkeyval.name = key 365 newkeyval.value = str(val) 366 367 368 def get_trivial_attr(self, obj, objdict): 369 """Get all trivial attributes from the object. 370 371 This function is used to extract attributes from a pb job. The 372 dictionary specifies the types of each attribute in each tko 373 class. 374 375 @param 376 obj: the pb object that is being extracted. 377 objdict: the dict that specifies the type. 378 379 @return a dict of each attr name and it's corresponding value. 380 """ 381 382 resultdict = {} 383 for field, field_type in objdict.items(): 384 value = getattr(obj, field) 385 if field_type in (str, int, long): 386 resultdict[field] = field_type(value) 387 elif field_type == datetime: 388 resultdict[field] = ( 389 datetime.fromtimestamp(value/1000.0)) 390 391 return resultdict 392 393 394 def set_trivial_attr(self, tko_obj, pb_obj, objdict): 395 """Sets all the easy attributes appropriately according to the 396 type. 397 398 This function is used to set all the trivial attributes 399 provided by objdict, the dictionary that specifies the types 400 of each attribute in each tko class. 401 402 @param 403 tko_obj: the original object that has the data being copied. 404 pb_obj: the new pb object that is being copied into. 405 objdict: specifies the type of each attribute in the class we 406 are working with. 407 408 """ 409 for attr, attr_type in objdict.iteritems(): 410 if attr_type == datetime: 411 t = getattr(tko_obj, attr) 412 if not t: 413 self.set_attr_safely(pb_obj, attr, t, int) 414 else: 415 t = mktime(t.timetuple()) + 1e-6 * t.microsecond 416 setattr(pb_obj, attr, long(t*1000)) 417 else: 418 value = getattr(tko_obj, attr) 419 self.set_attr_safely(pb_obj, attr, value, attr_type) 420 421 422 def set_attr_safely(self, var, attr, value, vartype): 423 """Sets a particular attribute of var if the provided value is 424 not None. 425 426 Checks if value is None. If not, set the attribute of the var 427 to be the default value. This is necessary for the special 428 required fields of the protocol buffer. 429 430 @param 431 var: the variable of which one of the attribute is being set. 432 attr: the attribute that is being set. 433 value: the value that is being checked 434 vartype: the expected type of the attr 435 436 """ 437 438 supported_types = [int, long, str] 439 if vartype in supported_types: 440 if value is None: 441 value = vartype() 442 elif not isinstance(value, vartype): 443 logging.warning('Unexpected type %s for attr %s, should be %s', 444 type(value), attr, vartype) 445 446 setattr(var, attr, vartype(value)) 447
