Home | History | Annotate | Download | only in cros 
      1 # Copyright 2017 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 import os
      5 import unittest
      6 
      7 import tradefed_test
      8 
      9 
     10 def _load_data(filename):
     11     """Loads the test data of the given file name."""
     12     with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
     13               'tradefed_test_unittest_data', filename), 'r') as f:
     14         return f.read()
     15 
     16 
     17 class TradefedTestTest(unittest.TestCase):
     18     """Unittest for tradefed_test."""
     19 
     20     def test_parse_tradefed_result(self):
     21         """Test for parse_tradefed_result."""
     22 
     23         waivers = set([
     24             'android.app.cts.SystemFeaturesTest#testUsbAccessory',
     25             'android.widget.cts.GridViewTest#testSetNumColumns',
     26         ])
     27 
     28         # b/35605415 and b/36520623
     29         # http://pantheon/storage/browser/chromeos-autotest-results/108103986-chromeos-test/
     30         # CTS: Tradefed may split a module to multiple chunks.
     31         # Besides, the module name may not end with "TestCases".
     32         self.assertEquals((35, 33, 2, 0, 0),
     33             tradefed_test.parse_tradefed_result(
     34                 _load_data('CtsHostsideNetworkTests.txt'),
     35                 waivers=waivers))
     36 
     37         # b/35530394
     38         # http://pantheon/storage/browser/chromeos-autotest-results/108291418-chromeos-test/
     39         # Crashed, but the automatic retry by tradefed executed the rest.
     40         self.assertEquals((1395, 1386, 9, 0, 0),
     41             tradefed_test.parse_tradefed_result(
     42                 _load_data('CtsMediaTestCases.txt'),
     43                 waivers=waivers))
     44 
     45         # b/35530394
     46         # http://pantheon/storage/browser/chromeos-autotest-results/106540705-chromeos-test/
     47         # Crashed in the middle, and the device didn't came back.
     48         self.assertEquals((110, 27, 1, 82, 0),
     49             tradefed_test.parse_tradefed_result(
     50                 _load_data('CtsSecurityTestCases.txt'),
     51                 waivers=waivers))
     52 
     53         # b/36629187
     54         # http://pantheon/storage/browser/chromeos-autotest-results/108855595-chromeos-test/
     55         # Crashed in the middle. Tradefed decided not to continue.
     56         self.assertEquals((739, 573, 3, 163, 0),
     57             tradefed_test.parse_tradefed_result(
     58                 _load_data('CtsViewTestCases.txt'),
     59                 waivers=waivers))
     60 
     61         # b/36375690
     62         # http://pantheon/storage/browser/chromeos-autotest-results/109040174-chromeos-test/
     63         # Mixture of real failures and waivers.
     64         self.assertEquals((321, 316, 5, 0, 1),
     65             tradefed_test.parse_tradefed_result(
     66                 _load_data('CtsAppTestCases.txt'),
     67                 waivers=waivers))
     68         # ... and the retry of the above failing iteration.
     69         self.assertEquals((5, 1, 4, 0, 1),
     70             tradefed_test.parse_tradefed_result(
     71                 _load_data('CtsAppTestCases-retry.txt'),
     72                 waivers=waivers))
     73 
     74         # http://pantheon/storage/browser/chromeos-autotest-results/116875512-chromeos-test/
     75         # When a test case crashed during teardown, tradefed prints the "fail"
     76         # message twice. Tolerate it and still return an (inconsistent) count.
     77         self.assertEquals((1194, 1185, 10, 0, 2),
     78             tradefed_test.parse_tradefed_result(
     79                 _load_data('CtsWidgetTestCases.txt'),
     80                 waivers=waivers))
     81 
     82         # http://pantheon/storage/browser/chromeos-autotest-results/117914707-chromeos-test/
     83         # When a test case unrecoverably crashed during teardown, tradefed
     84         # prints the "fail" and failure summary message twice. Tolerate it.
     85         self.assertEquals((71, 70, 1, 0, 0),
     86             tradefed_test.parse_tradefed_result(
     87                 _load_data('CtsPrintTestCases.txt'),
     88                 waivers=waivers))
     89 
     90         gts_waivers = set([
     91             ('com.google.android.placement.gts.CoreGmsAppsTest#' +
     92                 'testCoreGmsAppsPreloaded'),
     93             ('com.google.android.placement.gts.CoreGmsAppsTest#' +
     94                 'testGoogleDuoPreloaded'),
     95             'com.google.android.placement.gts.UiPlacementTest#testPlayStore'
     96         ])
     97 
     98         # crbug.com/748116
     99         # http://pantheon/storage/browser/chromeos-autotest-results/130080763-chromeos-test/
    100         # 3 ABIS: x86, x86_64, and armeabi-v7a
    101         self.assertEquals((15, 6, 9, 0, 9),
    102             tradefed_test.parse_tradefed_result(
    103                 _load_data('GtsPlacementTestCases.txt'),
    104                 waivers=gts_waivers))
    105 
    106         # b/64095702
    107         # http://pantheon/storage/browser/chromeos-autotest-results/130211812-chromeos-test/
    108         # The result of the last chunk not reported by tradefed.
    109         # The actual dEQP log is too big, hence the test data here is trimmed.
    110         self.assertEquals((157871, 116916, 0, 40955, 0),
    111             tradefed_test.parse_tradefed_result(
    112                 _load_data('CtsDeqpTestCases-trimmed.txt'),
    113                 waivers=waivers))
    114 
    115 if __name__ == '__main__':
    116     unittest.main()
    117