Home | History | Annotate | Download | only in tools 
      1 # Copyright 2014 The Android Open Source Project
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
      4 # you may not use this file except in compliance with the License.
      5 # You may obtain a copy of the License at
      6 #
      7 #      http://www.apache.org/licenses/LICENSE-2.0
      8 #
      9 # Unless required by applicable law or agreed to in writing, software
     10 # distributed under the License is distributed on an "AS IS" BASIS,
     11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     12 # See the License for the specific language governing permissions and
     13 # limitations under the License.
     14 
     15 import os
     16 import os.path
     17 import tempfile
     18 import subprocess
     19 import time
     20 import sys
     21 import its.device
     22 
     23 def main():
     24     """Run all the automated tests, saving intermediate files, and producing
     25     a summary/report of the results.
     26 
     27     Script should be run from the top-level CameraITS directory.
     28     """
     29 
     30     SKIP_RET_CODE = 101
     31 
     32     # Not yet mandated tests
     33     NOT_YET_MANDATED = {
     34         "scene0":[
     35             "test_jitter"
     36         ],
     37         "scene1":[
     38             "test_ae_precapture_trigger",
     39             "test_crop_region_raw",
     40             "test_ev_compensation_advanced",
     41             "test_ev_compensation_basic",
     42             "test_yuv_plus_jpeg"
     43         ]
     44     }
     45 
     46     # Get all the scene0 and scene1 tests, which can be run using the same
     47     # physical setup.
     48     scenes = ["scene0", "scene1"]
     49     tests = []
     50     for d in scenes:
     51         tests += [(d,s[:-3],os.path.join("tests", d, s))
     52                   for s in os.listdir(os.path.join("tests",d))
     53                   if s[-3:] == ".py"]
     54     tests.sort()
     55 
     56     # Make output directories to hold the generated files.
     57     topdir = tempfile.mkdtemp()
     58     for d in scenes:
     59         os.mkdir(os.path.join(topdir, d))
     60     print "Saving output files to:", topdir, "\n"
     61 
     62     # determine camera id
     63     camera_id = 0
     64     for s in sys.argv[1:]:
     65         if s[:7] == "camera=" and len(s) > 7:
     66             camera_id = s[7:]
     67 
     68     # Run each test, capturing stdout and stderr.
     69     numpass = 0
     70     numskip = 0
     71     numnotmandatedfail = 0
     72     numfail = 0
     73     for (scene,testname,testpath) in tests:
     74         cmd = ['python', os.path.join(os.getcwd(),testpath)] + sys.argv[1:]
     75         outdir = os.path.join(topdir,scene)
     76         outpath = os.path.join(outdir,testname+"_stdout.txt")
     77         errpath = os.path.join(outdir,testname+"_stderr.txt")
     78         t0 = time.time()
     79         with open(outpath,"w") as fout, open(errpath,"w") as ferr:
     80             retcode = subprocess.call(cmd,stderr=ferr,stdout=fout,cwd=outdir)
     81         t1 = time.time()
     82 
     83         if retcode == 0:
     84             retstr = "PASS "
     85             numpass += 1
     86         elif retcode == SKIP_RET_CODE:
     87             retstr = "SKIP "
     88             numskip += 1
     89         elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
     90             retstr = "FAIL*"
     91             numnotmandatedfail += 1
     92         else:
     93             retstr = "FAIL "
     94             numfail += 1
     95 
     96         print "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
     97 
     98     if numskip > 0:
     99         skipstr = ", %d test%s skipped" % (numskip, "s" if numskip > 1 else "")
    100     else:
    101         skipstr = ""
    102 
    103     print "\n%d / %d tests passed (%.1f%%)%s" % (
    104             numpass + numnotmandatedfail, len(tests) - numskip,
    105             100.0 * float(numpass + numnotmandatedfail) / (len(tests) - numskip)
    106                 if len(tests) != numskip else 100.0,
    107             skipstr)
    108 
    109     if numnotmandatedfail > 0:
    110         print "(*) tests are not yet mandated"
    111 
    112     its.device.report_result(camera_id, numfail == 0)
    113 
    114 if __name__ == '__main__':
    115     main()
    116 
    117