1 # Copyright 2014 The Android Open Source Project 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 import copy 16 import os 17 import os.path 18 import tempfile 19 import subprocess 20 import time 21 import sys 22 23 import its.caps 24 import its.device 25 from its.device import ItsSession 26 27 CHART_DELAY = 1 # seconds 28 FACING_EXTERNAL = 2 29 NUM_TRYS = 2 30 SKIP_RET_CODE = 101 # note this must be same as tests/scene*/test_* 31 32 33 def evaluate_socket_failure(err_file_path): 34 """Determine if test fails due to socket FAIL.""" 35 socket_fail = False 36 with open(err_file_path, 'r') as ferr: 37 for line in ferr: 38 if (line.find('socket.error') != -1 or 39 line.find('Problem with socket') != -1): 40 socket_fail = True 41 return socket_fail 42 43 44 def skip_sensor_fusion(): 45 """Determine if sensor fusion test is skipped for this camera.""" 46 47 skip_code = SKIP_RET_CODE 48 with ItsSession() as cam: 49 props = cam.get_camera_properties() 50 if (its.caps.sensor_fusion(props) and its.caps.manual_sensor(props) and 51 props['android.lens.facing'] is not FACING_EXTERNAL): 52 skip_code = None 53 return skip_code 54 55 56 def main(): 57 """Run all the automated tests, saving intermediate files, and producing 58 a summary/report of the results. 59 60 Script should be run from the top-level CameraITS directory. 61 62 Command line arguments: 63 camera: the camera(s) to be tested. Use comma to separate multiple 64 camera Ids. Ex: "camera=0,1" or "camera=1" 65 device: device id for adb 66 scenes: the test scene(s) to be executed. Use comma to separate 67 multiple scenes. Ex: "scenes=scene0,scene1" or 68 "scenes=0,1,sensor_fusion" (sceneX can be abbreviated by X 69 where X is a integer) 70 chart: [Experimental] another android device served as test chart 71 display. When this argument presents, change of test scene 72 will be handled automatically. Note that this argument 73 requires special physical/hardware setup to work and may not 74 work on all android devices. 75 result: Device ID to forward results to (in addition to the device 76 that the tests are running on). 77 rot_rig: [Experimental] ID of the rotation rig being used (formatted as 78 "<vendor ID>:<product ID>:<channel #>" or "default") 79 tmp_dir: location of temp directory for output files 80 skip_scene_validation: force skip scene validation. Used when test scene 81 is setup up front and don't require tester validation. 82 """ 83 84 # Not yet mandated tests 85 NOT_YET_MANDATED = { 86 "scene0": [ 87 "test_jitter", 88 "test_burst_capture" 89 ], 90 "scene1": [ 91 "test_ae_af", 92 "test_ae_precapture_trigger", 93 "test_crop_region_raw", 94 "test_ev_compensation_advanced", 95 "test_ev_compensation_basic", 96 "test_yuv_plus_jpeg" 97 ], 98 "scene2": [ 99 "test_num_faces", 100 ], 101 "scene3": [ 102 "test_3a_consistency", 103 "test_lens_movement_reporting", 104 "test_lens_position" 105 ], 106 "scene4": [], 107 "scene5": [], 108 "sensor_fusion": [] 109 } 110 111 all_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5", 112 "sensor_fusion"] 113 114 auto_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4"] 115 116 scene_req = { 117 "scene0": None, 118 "scene1": "A grey card covering at least the middle 30% of the scene", 119 "scene2": "A picture containing human faces", 120 "scene3": "The ISO 12233 chart", 121 "scene4": "A specific test page of a circle covering at least the " 122 "middle 50% of the scene. See CameraITS.pdf section 2.3.4 " 123 "for more details", 124 "scene5": "Capture images with a diffuser attached to the camera. See " 125 "CameraITS.pdf section 2.3.4 for more details", 126 "sensor_fusion": "Rotating checkboard pattern. See " 127 "sensor_fusion/SensorFusion.pdf for detailed " 128 "instructions.\nNote that this test will be skipped " 129 "on devices not supporting REALTIME camera timestamp." 130 } 131 scene_extra_args = { 132 "scene5": ["doAF=False"] 133 } 134 135 camera_ids = [] 136 scenes = [] 137 chart_host_id = None 138 result_device_id = None 139 rot_rig_id = None 140 tmp_dir = None 141 skip_scene_validation = False 142 for s in sys.argv[1:]: 143 if s[:7] == "camera=" and len(s) > 7: 144 camera_ids = s[7:].split(',') 145 elif s[:7] == "scenes=" and len(s) > 7: 146 scenes = s[7:].split(',') 147 elif s[:6] == 'chart=' and len(s) > 6: 148 chart_host_id = s[6:] 149 elif s[:7] == 'result=' and len(s) > 7: 150 result_device_id = s[7:] 151 elif s[:8] == 'rot_rig=' and len(s) > 8: 152 rot_rig_id = s[8:] # valid values: 'default' or '$VID:$PID:$CH' 153 # The default '$VID:$PID:$CH' is '04d8:fc73:1' 154 elif s[:8] == 'tmp_dir=' and len(s) > 8: 155 tmp_dir = s[8:] 156 elif s == 'skip_scene_validation': 157 skip_scene_validation = True 158 159 auto_scene_switch = chart_host_id is not None 160 merge_result_switch = result_device_id is not None 161 162 # Run through all scenes if user does not supply one 163 possible_scenes = auto_scenes if auto_scene_switch else all_scenes 164 if not scenes: 165 scenes = possible_scenes 166 else: 167 # Validate user input scene names 168 valid_scenes = True 169 temp_scenes = [] 170 for s in scenes: 171 if s in possible_scenes: 172 temp_scenes.append(s) 173 else: 174 try: 175 # Try replace "X" to "sceneX" 176 scene_str = "scene" + s 177 if scene_str not in possible_scenes: 178 valid_scenes = False 179 break 180 temp_scenes.append(scene_str) 181 except ValueError: 182 valid_scenes = False 183 break 184 185 if not valid_scenes: 186 print "Unknown scene specifiied:", s 187 assert False 188 scenes = temp_scenes 189 190 # Initialize test results 191 results = {} 192 result_key = ItsSession.RESULT_KEY 193 for s in all_scenes: 194 results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED} 195 196 # Make output directories to hold the generated files. 197 topdir = tempfile.mkdtemp(dir=tmp_dir) 198 subprocess.call(['chmod', 'g+rx', topdir]) 199 print "Saving output files to:", topdir, "\n" 200 201 device_id = its.device.get_device_id() 202 device_id_arg = "device=" + device_id 203 print "Testing device " + device_id 204 205 # Sanity Check for devices 206 device_bfp = its.device.get_device_fingerprint(device_id) 207 assert device_bfp is not None 208 209 if auto_scene_switch: 210 chart_host_bfp = its.device.get_device_fingerprint(chart_host_id) 211 assert chart_host_bfp is not None 212 213 if merge_result_switch: 214 result_device_bfp = its.device.get_device_fingerprint(result_device_id) 215 assert_err_msg = ('Cannot merge result to a different build, from ' 216 '%s to %s' % (device_bfp, result_device_bfp)) 217 assert device_bfp == result_device_bfp, assert_err_msg 218 219 # user doesn't specify camera id, run through all cameras 220 if not camera_ids: 221 camera_ids_path = os.path.join(topdir, "camera_ids.txt") 222 out_arg = "out=" + camera_ids_path 223 cmd = ['python', 224 os.path.join(os.getcwd(), "tools/get_camera_ids.py"), out_arg, 225 device_id_arg] 226 cam_code = subprocess.call(cmd, cwd=topdir) 227 assert cam_code == 0 228 with open(camera_ids_path, "r") as f: 229 for line in f: 230 camera_ids.append(line.replace('\n', '')) 231 232 print "Running ITS on camera: %s, scene %s" % (camera_ids, scenes) 233 234 if auto_scene_switch: 235 # merge_result only supports run_parallel_tests 236 if merge_result_switch and camera_ids[0] == '1': 237 print 'Skip chart screen' 238 time.sleep(1) 239 else: 240 print 'Waking up chart screen: ', chart_host_id 241 screen_id_arg = ('screen=%s' % chart_host_id) 242 cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 243 'wake_up_screen.py'), screen_id_arg] 244 wake_code = subprocess.call(cmd) 245 assert wake_code == 0 246 247 for camera_id in camera_ids: 248 # Loop capturing images until user confirm test scene is correct 249 camera_id_arg = "camera=" + camera_id 250 print "Preparing to run ITS on camera", camera_id 251 252 os.mkdir(os.path.join(topdir, camera_id)) 253 for d in scenes: 254 os.mkdir(os.path.join(topdir, camera_id, d)) 255 256 for scene in scenes: 257 skip_code = None 258 tests = [(s[:-3], os.path.join("tests", scene, s)) 259 for s in os.listdir(os.path.join("tests", scene)) 260 if s[-3:] == ".py" and s[:4] == "test"] 261 tests.sort() 262 263 summary = "Cam" + camera_id + " " + scene + "\n" 264 numpass = 0 265 numskip = 0 266 num_not_mandated_fail = 0 267 numfail = 0 268 validate_switch = True 269 if scene_req[scene] is not None: 270 out_path = os.path.join(topdir, camera_id, scene+".jpg") 271 out_arg = "out=" + out_path 272 if scene == 'sensor_fusion': 273 skip_code = skip_sensor_fusion() 274 if rot_rig_id or skip_code == SKIP_RET_CODE: 275 validate_switch = False 276 if skip_scene_validation: 277 validate_switch = False 278 cmd = None 279 if auto_scene_switch: 280 if (not merge_result_switch or 281 (merge_result_switch and camera_ids[0] == '0')): 282 scene_arg = 'scene=' + scene 283 cmd = ['python', 284 os.path.join(os.getcwd(), 'tools/load_scene.py'), 285 scene_arg, screen_id_arg] 286 else: 287 time.sleep(CHART_DELAY) 288 else: 289 # Skip scene validation under certain conditions 290 if validate_switch and not merge_result_switch: 291 scene_arg = 'scene=' + scene_req[scene] 292 extra_args = scene_extra_args.get(scene, []) 293 cmd = ['python', 294 os.path.join(os.getcwd(), 295 'tools/validate_scene.py'), 296 camera_id_arg, out_arg, 297 scene_arg, device_id_arg] + extra_args 298 if cmd is not None: 299 valid_scene_code = subprocess.call(cmd, cwd=topdir) 300 assert valid_scene_code == 0 301 print "Start running ITS on camera %s, %s" % (camera_id, scene) 302 # Run each test, capturing stdout and stderr. 303 for (testname, testpath) in tests: 304 if auto_scene_switch: 305 if merge_result_switch and camera_ids[0] == '0': 306 # Send an input event to keep the screen not dimmed. 307 # Since we are not using camera of chart screen, FOCUS event 308 # should do nothing but keep the screen from dimming. 309 # The "sleep after x minutes of inactivity" display setting 310 # determines how long this command can keep screen bright. 311 # Setting it to something like 30 minutes should be enough. 312 cmd = ('adb -s %s shell input keyevent FOCUS' 313 % chart_host_id) 314 subprocess.call(cmd.split()) 315 t0 = time.time() 316 for num_try in range(NUM_TRYS): 317 outdir = os.path.join(topdir, camera_id, scene) 318 outpath = os.path.join(outdir, testname+'_stdout.txt') 319 errpath = os.path.join(outdir, testname+'_stderr.txt') 320 if scene == 'sensor_fusion': 321 if skip_code is not SKIP_RET_CODE: 322 if rot_rig_id: 323 print 'Rotating phone w/ rig %s' % rot_rig_id 324 rig = ('python tools/rotation_rig.py rotator=%s' % 325 rot_rig_id) 326 subprocess.Popen(rig.split()) 327 else: 328 print 'Rotate phone 15s as shown in SensorFusion.pdf' 329 else: 330 test_code = skip_code 331 if skip_code is not SKIP_RET_CODE: 332 cmd = ['python', os.path.join(os.getcwd(), testpath)] 333 cmd += sys.argv[1:] + [camera_id_arg] 334 with open(outpath, 'w') as fout, open(errpath, 'w') as ferr: 335 test_code = subprocess.call( 336 cmd, stderr=ferr, stdout=fout, cwd=outdir) 337 if test_code == 0 or test_code == SKIP_RET_CODE: 338 break 339 else: 340 socket_fail = evaluate_socket_failure(errpath) 341 if socket_fail: 342 if num_try != NUM_TRYS-1: 343 print ' Retry %s/%s' % (scene, testname) 344 else: 345 break 346 else: 347 break 348 t1 = time.time() 349 350 test_failed = False 351 if test_code == 0: 352 retstr = "PASS " 353 numpass += 1 354 elif test_code == SKIP_RET_CODE: 355 retstr = "SKIP " 356 numskip += 1 357 elif test_code != 0 and testname in NOT_YET_MANDATED[scene]: 358 retstr = "FAIL*" 359 num_not_mandated_fail += 1 360 else: 361 retstr = "FAIL " 362 numfail += 1 363 test_failed = True 364 365 msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0) 366 print msg 367 msg_short = "%s %s [%.1fs]" % (retstr, testname, t1-t0) 368 if test_failed: 369 summary += msg_short + "\n" 370 371 if numskip > 0: 372 skipstr = ", %d test%s skipped" % ( 373 numskip, "s" if numskip > 1 else "") 374 else: 375 skipstr = "" 376 377 test_result = "\n%d / %d tests passed (%.1f%%)%s" % ( 378 numpass + num_not_mandated_fail, len(tests) - numskip, 379 100.0 * float(numpass + num_not_mandated_fail) / 380 (len(tests) - numskip) 381 if len(tests) != numskip else 100.0, skipstr) 382 print test_result 383 384 if num_not_mandated_fail > 0: 385 msg = "(*) tests are not yet mandated" 386 print msg 387 388 summary_path = os.path.join(topdir, camera_id, scene, "summary.txt") 389 with open(summary_path, "w") as f: 390 f.write(summary) 391 392 passed = numfail == 0 393 results[scene][result_key] = (ItsSession.RESULT_PASS if passed 394 else ItsSession.RESULT_FAIL) 395 results[scene][ItsSession.SUMMARY_KEY] = summary_path 396 397 print "Reporting ITS result to CtsVerifier" 398 if merge_result_switch: 399 # results are modified by report_result 400 results_backup = copy.deepcopy(results) 401 its.device.report_result(result_device_id, camera_id, results_backup) 402 403 its.device.report_result(device_id, camera_id, results) 404 405 if auto_scene_switch: 406 if merge_result_switch: 407 print 'Skip shutting down chart screen' 408 else: 409 print 'Shutting down chart screen: ', chart_host_id 410 screen_id_arg = ('screen=%s' % chart_host_id) 411 cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 412 'turn_off_screen.py'), screen_id_arg] 413 screen_off_code = subprocess.call(cmd) 414 assert screen_off_code == 0 415 416 print 'Shutting down DUT screen: ', device_id 417 screen_id_arg = ('screen=%s' % device_id) 418 cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 419 'turn_off_screen.py'), screen_id_arg] 420 screen_off_code = subprocess.call(cmd) 421 assert screen_off_code == 0 422 423 print "ITS tests finished. Please go back to CtsVerifier and proceed" 424 425 if __name__ == '__main__': 426 main() 427
