Home | History | Annotate | Download | only in cros 
      1 # Copyright 2017 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 import common
      5 import os
      6 import unittest
      7 
      8 import tradefed_test
      9 
     10 
     11 def _load_data(filename):
     12     """Loads the test data of the given file name."""
     13     with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
     14                            'tradefed_test_unittest_data', filename), 'r') as f:
     15         return f.read()
     16 
     17 
     18 class TradefedTestTest(unittest.TestCase):
     19     """Unittest for tradefed_test."""
     20 
     21     def test_parse_tradefed_result(self):
     22         """Test for parse_tradefed_result."""
     23 
     24         waivers = set([
     25             'android.app.cts.SystemFeaturesTest#testUsbAccessory',
     26             'android.widget.cts.GridViewTest#testSetNumColumns',
     27         ])
     28 
     29         # b/35605415 and b/36520623
     30         # http://pantheon/storage/browser/chromeos-autotest-results/108103986-chromeos-test/
     31         # CTS: Tradefed may split a module to multiple chunks.
     32         # Besides, the module name may not end with "TestCases".
     33         waived, _ = tradefed_test.parse_tradefed_result(
     34             _load_data('CtsHostsideNetworkTests.txt'),
     35             waivers=waivers)
     36         self.assertEquals(0, len(waived))
     37 
     38         # b/35530394
     39         # http://pantheon/storage/browser/chromeos-autotest-results/108291418-chromeos-test/
     40         # Crashed, but the automatic retry by tradefed executed the rest.
     41         waived, _ = tradefed_test.parse_tradefed_result(
     42             _load_data('CtsMediaTestCases.txt'),
     43             waivers=waivers)
     44         self.assertEquals(0, len(waived))
     45 
     46         # b/35530394
     47         # http://pantheon/storage/browser/chromeos-autotest-results/106540705-chromeos-test/
     48         # Crashed in the middle, and the device didn't came back.
     49         waived, _ = tradefed_test.parse_tradefed_result(
     50             _load_data('CtsSecurityTestCases.txt'),
     51             waivers=waivers)
     52         self.assertEquals(0, len(waived))
     53 
     54         # b/36629187
     55         # http://pantheon/storage/browser/chromeos-autotest-results/108855595-chromeos-test/
     56         # Crashed in the middle. Tradefed decided not to continue.
     57         waived, _ = tradefed_test.parse_tradefed_result(
     58             _load_data('CtsViewTestCases.txt'),
     59             waivers=waivers)
     60         self.assertEquals(0, len(waived))
     61 
     62         # b/36375690
     63         # http://pantheon/storage/browser/chromeos-autotest-results/109040174-chromeos-test/
     64         # Mixture of real failures and waivers.
     65         waived, _ = tradefed_test.parse_tradefed_result(
     66             _load_data('CtsAppTestCases.txt'),
     67             waivers=waivers)
     68         self.assertEquals(1, len(waived))
     69         # ... and the retry of the above failing iteration.
     70         waived, _ = tradefed_test.parse_tradefed_result(
     71             _load_data('CtsAppTestCases-retry.txt'),
     72             waivers=waivers)
     73         self.assertEquals(1, len(waived))
     74 
     75         # http://pantheon/storage/browser/chromeos-autotest-results/116875512-chromeos-test/
     76         # When a test case crashed during teardown, tradefed prints the "fail"
     77         # message twice. Tolerate it and still return an (inconsistent) count.
     78         waived, _ = tradefed_test.parse_tradefed_result(
     79             _load_data('CtsWidgetTestCases.txt'),
     80             waivers=waivers)
     81         self.assertEquals(1, len(waived))
     82 
     83         # http://pantheon/storage/browser/chromeos-autotest-results/117914707-chromeos-test/
     84         # When a test case unrecoverably crashed during teardown, tradefed
     85         # prints the "fail" and failure summary message twice. Tolerate it.
     86         waived, _ = tradefed_test.parse_tradefed_result(
     87             _load_data('CtsPrintTestCases.txt'),
     88             waivers=waivers)
     89         self.assertEquals(0, len(waived))
     90 
     91         gts_waivers = set([
     92             ('com.google.android.placement.gts.CoreGmsAppsTest#' +
     93                 'testCoreGmsAppsPreloaded'),
     94             ('com.google.android.placement.gts.CoreGmsAppsTest#' +
     95                 'testGoogleDuoPreloaded'),
     96             'com.google.android.placement.gts.UiPlacementTest#testPlayStore'
     97         ])
     98 
     99         # crbug.com/748116
    100         # http://pantheon/storage/browser/chromeos-autotest-results/130080763-chromeos-test/
    101         # 3 ABIS: x86, x86_64, and armeabi-v7a
    102         waived, _ = tradefed_test.parse_tradefed_result(
    103             _load_data('GtsPlacementTestCases.txt'),
    104             waivers=gts_waivers)
    105         self.assertEquals(9, len(waived))
    106 
    107         # b/64095702
    108         # http://pantheon/storage/browser/chromeos-autotest-results/130211812-chromeos-test/
    109         # The result of the last chunk not reported by tradefed.
    110         # The actual dEQP log is too big, hence the test data here is trimmed.
    111         waived, _ = tradefed_test.parse_tradefed_result(
    112             _load_data('CtsDeqpTestCases-trimmed.txt'),
    113             waivers=waivers)
    114         self.assertEquals(0, len(waived))
    115 
    116         # b/80160772
    117         # http://pantheon/storage/browser/chromeos-autotest-results/201962931-kkanchi/
    118         # The newer tradefed requires different parsing to count waivers.
    119         waived, _ = tradefed_test.parse_tradefed_result(
    120             _load_data('CtsAppTestCases_P_simplified.txt'),
    121             waivers=waivers)
    122         self.assertEquals(1, len(waived))
    123 
    124         # b/66899135, tradefed may reported inaccuratly with `list results`.
    125         # Check if summary section shows that the result is inacurrate.
    126         _, accurate = tradefed_test.parse_tradefed_result(
    127             _load_data('CtsAppTestCases_P_simplified.txt'),
    128             waivers=waivers)
    129         self.assertTrue(accurate)
    130 
    131         _, accurate = tradefed_test.parse_tradefed_result(
    132             _load_data('CtsDeqpTestCases-trimmed-inaccurate.txt'),
    133             waivers=waivers)
    134         self.assertFalse(accurate)
    135 
    136 if __name__ == '__main__':
    137     unittest.main()
    138