1 #!/usr/bin/env python 2 # 3 # Copyright 2006, Google Inc. 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions are 8 # met: 9 # 10 # * Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # * Redistributions in binary form must reproduce the above 13 # copyright notice, this list of conditions and the following disclaimer 14 # in the documentation and/or other materials provided with the 15 # distribution. 16 # * Neither the name of Google Inc. nor the names of its 17 # contributors may be used to endorse or promote products derived from 18 # this software without specific prior written permission. 19 # 20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32 """Unit test for the gtest_xml_output module""" 33 34 __author__ = 'eefacm (at] gmail.com (Sean Mcafee)' 35 36 import datetime 37 import errno 38 import os 39 import re 40 import sys 41 from xml.dom import minidom, Node 42 43 import gtest_test_utils 44 import gtest_xml_test_utils 45 46 47 GTEST_FILTER_FLAG = '--gtest_filter' 48 GTEST_LIST_TESTS_FLAG = '--gtest_list_tests' 49 GTEST_OUTPUT_FLAG = "--gtest_output" 50 GTEST_DEFAULT_OUTPUT_FILE = "test_detail.xml" 51 GTEST_PROGRAM_NAME = "gtest_xml_output_unittest_" 52 53 SUPPORTS_STACK_TRACES = False 54 55 if SUPPORTS_STACK_TRACES: 56 STACK_TRACE_TEMPLATE = '\nStack trace:\n*' 57 else: 58 STACK_TRACE_TEMPLATE = '' 59 60 EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?> 61 <testsuites tests="23" failures="4" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42"> 62 <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" errors="0" time="*"> 63 <testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/> 64 </testsuite> 65 <testsuite name="FailedTest" tests="1" failures="1" disabled="0" errors="0" time="*"> 66 <testcase name="Fails" status="run" time="*" classname="FailedTest"> 67 <failure message="gtest_xml_output_unittest_.cc:*
Value of: 2
Expected: 1" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 68 Value of: 2 69 Expected: 1%(stack)s]]></failure> 70 </testcase> 71 </testsuite> 72 <testsuite name="MixedResultTest" tests="3" failures="1" disabled="1" errors="0" time="*"> 73 <testcase name="Succeeds" status="run" time="*" classname="MixedResultTest"/> 74 <testcase name="Fails" status="run" time="*" classname="MixedResultTest"> 75 <failure message="gtest_xml_output_unittest_.cc:*
Value of: 2
Expected: 1" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 76 Value of: 2 77 Expected: 1%(stack)s]]></failure> 78 <failure message="gtest_xml_output_unittest_.cc:*
Value of: 3
Expected: 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 79 Value of: 3 80 Expected: 2%(stack)s]]></failure> 81 </testcase> 82 <testcase name="DISABLED_test" status="notrun" time="*" classname="MixedResultTest"/> 83 </testsuite> 84 <testsuite name="XmlQuotingTest" tests="1" failures="1" disabled="0" errors="0" time="*"> 85 <testcase name="OutputsCData" status="run" time="*" classname="XmlQuotingTest"> 86 <failure message="gtest_xml_output_unittest_.cc:*
Failed
XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]></top>" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 87 Failed 88 XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]>]]><![CDATA[</top>%(stack)s]]></failure> 89 </testcase> 90 </testsuite> 91 <testsuite name="InvalidCharactersTest" tests="1" failures="1" disabled="0" errors="0" time="*"> 92 <testcase name="InvalidCharactersInMessage" status="run" time="*" classname="InvalidCharactersTest"> 93 <failure message="gtest_xml_output_unittest_.cc:*
Failed
Invalid characters in brackets []" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 94 Failed 95 Invalid characters in brackets []%(stack)s]]></failure> 96 </testcase> 97 </testsuite> 98 <testsuite name="DisabledTest" tests="1" failures="0" disabled="1" errors="0" time="*"> 99 <testcase name="DISABLED_test_not_run" status="notrun" time="*" classname="DisabledTest"/> 100 </testsuite> 101 <testsuite name="PropertyRecordingTest" tests="4" failures="0" disabled="0" errors="0" time="*" SetUpTestCase="yes" TearDownTestCase="aye"> 102 <testcase name="OneProperty" status="run" time="*" classname="PropertyRecordingTest" key_1="1"/> 103 <testcase name="IntValuedProperty" status="run" time="*" classname="PropertyRecordingTest" key_int="1"/> 104 <testcase name="ThreeProperties" status="run" time="*" classname="PropertyRecordingTest" key_1="1" key_2="2" key_3="3"/> 105 <testcase name="TwoValuesForOneKeyUsesLastValue" status="run" time="*" classname="PropertyRecordingTest" key_1="2"/> 106 </testsuite> 107 <testsuite name="NoFixtureTest" tests="3" failures="0" disabled="0" errors="0" time="*"> 108 <testcase name="RecordProperty" status="run" time="*" classname="NoFixtureTest" key="1"/> 109 <testcase name="ExternalUtilityThatCallsRecordIntValuedProperty" status="run" time="*" classname="NoFixtureTest" key_for_utility_int="1"/> 110 <testcase name="ExternalUtilityThatCallsRecordStringValuedProperty" status="run" time="*" classname="NoFixtureTest" key_for_utility_string="1"/> 111 </testsuite> 112 <testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" errors="0" time="*"> 113 <testcase name="HasValueParamAttribute/0" value_param="33" status="run" time="*" classname="Single/ValueParamTest" /> 114 <testcase name="HasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" /> 115 <testcase name="AnotherTestThatHasValueParamAttribute/0" value_param="33" status="run" time="*" classname="Single/ValueParamTest" /> 116 <testcase name="AnotherTestThatHasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" /> 117 </testsuite> 118 <testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" errors="0" time="*"> 119 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="TypedTest/0" /> 120 </testsuite> 121 <testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" errors="0" time="*"> 122 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="TypedTest/1" /> 123 </testsuite> 124 <testsuite name="Single/TypeParameterizedTestCase/0" tests="1" failures="0" disabled="0" errors="0" time="*"> 125 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="Single/TypeParameterizedTestCase/0" /> 126 </testsuite> 127 <testsuite name="Single/TypeParameterizedTestCase/1" tests="1" failures="0" disabled="0" errors="0" time="*"> 128 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="Single/TypeParameterizedTestCase/1" /> 129 </testsuite> 130 </testsuites>""" % {'stack': STACK_TRACE_TEMPLATE} 131 132 EXPECTED_FILTERED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?> 133 <testsuites tests="1" failures="0" disabled="0" errors="0" time="*" 134 timestamp="*" name="AllTests" ad_hoc_property="42"> 135 <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" 136 errors="0" time="*"> 137 <testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/> 138 </testsuite> 139 </testsuites>""" 140 141 EXPECTED_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?> 142 <testsuites tests="0" failures="0" disabled="0" errors="0" time="*" 143 timestamp="*" name="AllTests"> 144 </testsuites>""" 145 146 GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME) 147 148 SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess( 149 [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output 150 151 152 class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase): 153 """ 154 Unit test for Google Test's XML output functionality. 155 """ 156 157 # This test currently breaks on platforms that do not support typed and 158 # type-parameterized tests, so we don't run it under them. 159 if SUPPORTS_TYPED_TESTS: 160 def testNonEmptyXmlOutput(self): 161 """ 162 Runs a test program that generates a non-empty XML output, and 163 tests that the XML output is expected. 164 """ 165 self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY_XML, 1) 166 167 def testEmptyXmlOutput(self): 168 """Verifies XML output for a Google Test binary without actual tests. 169 170 Runs a test program that generates an empty XML output, and 171 tests that the XML output is expected. 172 """ 173 174 self._TestXmlOutput('gtest_no_test_unittest', EXPECTED_EMPTY_XML, 0) 175 176 def testTimestampValue(self): 177 """Checks whether the timestamp attribute in the XML output is valid. 178 179 Runs a test program that generates an empty XML output, and checks if 180 the timestamp attribute in the testsuites tag is valid. 181 """ 182 actual = self._GetXmlOutput('gtest_no_test_unittest', [], 0) 183 date_time_str = actual.documentElement.getAttributeNode('timestamp').value 184 # datetime.strptime() is only available in Python 2.5+ so we have to 185 # parse the expected datetime manually. 186 match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str) 187 self.assertTrue( 188 re.match, 189 'XML datettime string %s has incorrect format' % date_time_str) 190 date_time_from_xml = datetime.datetime( 191 year=int(match.group(1)), month=int(match.group(2)), 192 day=int(match.group(3)), hour=int(match.group(4)), 193 minute=int(match.group(5)), second=int(match.group(6))) 194 195 time_delta = abs(datetime.datetime.now() - date_time_from_xml) 196 # timestamp value should be near the current local time 197 self.assertTrue(time_delta < datetime.timedelta(seconds=600), 198 'time_delta is %s' % time_delta) 199 actual.unlink() 200 201 def testDefaultOutputFile(self): 202 """ 203 Confirms that Google Test produces an XML output file with the expected 204 default name if no name is explicitly specified. 205 """ 206 output_file = os.path.join(gtest_test_utils.GetTempDir(), 207 GTEST_DEFAULT_OUTPUT_FILE) 208 gtest_prog_path = gtest_test_utils.GetTestExecutablePath( 209 'gtest_no_test_unittest') 210 try: 211 os.remove(output_file) 212 except OSError, e: 213 if e.errno != errno.ENOENT: 214 raise 215 216 p = gtest_test_utils.Subprocess( 217 [gtest_prog_path, '%s=xml' % GTEST_OUTPUT_FLAG], 218 working_dir=gtest_test_utils.GetTempDir()) 219 self.assert_(p.exited) 220 self.assertEquals(0, p.exit_code) 221 self.assert_(os.path.isfile(output_file)) 222 223 def testSuppressedXmlOutput(self): 224 """ 225 Tests that no XML file is generated if the default XML listener is 226 shut down before RUN_ALL_TESTS is invoked. 227 """ 228 229 xml_path = os.path.join(gtest_test_utils.GetTempDir(), 230 GTEST_PROGRAM_NAME + 'out.xml') 231 if os.path.isfile(xml_path): 232 os.remove(xml_path) 233 234 command = [GTEST_PROGRAM_PATH, 235 '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path), 236 '--shut_down_xml'] 237 p = gtest_test_utils.Subprocess(command) 238 if p.terminated_by_signal: 239 # p.signal is avalable only if p.terminated_by_signal is True. 240 self.assertFalse( 241 p.terminated_by_signal, 242 '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal)) 243 else: 244 self.assert_(p.exited) 245 self.assertEquals(1, p.exit_code, 246 "'%s' exited with code %s, which doesn't match " 247 'the expected exit code %s.' 248 % (command, p.exit_code, 1)) 249 250 self.assert_(not os.path.isfile(xml_path)) 251 252 def testFilteredTestXmlOutput(self): 253 """Verifies XML output when a filter is applied. 254 255 Runs a test program that executes only some tests and verifies that 256 non-selected tests do not show up in the XML output. 257 """ 258 259 self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED_TEST_XML, 0, 260 extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG]) 261 262 def _GetXmlOutput(self, gtest_prog_name, extra_args, expected_exit_code): 263 """ 264 Returns the xml output generated by running the program gtest_prog_name. 265 Furthermore, the program's exit code must be expected_exit_code. 266 """ 267 xml_path = os.path.join(gtest_test_utils.GetTempDir(), 268 gtest_prog_name + 'out.xml') 269 gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name) 270 271 command = ([gtest_prog_path, '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path)] + 272 extra_args) 273 p = gtest_test_utils.Subprocess(command) 274 if p.terminated_by_signal: 275 self.assert_(False, 276 '%s was killed by signal %d' % (gtest_prog_name, p.signal)) 277 else: 278 self.assert_(p.exited) 279 self.assertEquals(expected_exit_code, p.exit_code, 280 "'%s' exited with code %s, which doesn't match " 281 'the expected exit code %s.' 282 % (command, p.exit_code, expected_exit_code)) 283 actual = minidom.parse(xml_path) 284 return actual 285 286 def _TestXmlOutput(self, gtest_prog_name, expected_xml, 287 expected_exit_code, extra_args=None): 288 """ 289 Asserts that the XML document generated by running the program 290 gtest_prog_name matches expected_xml, a string containing another 291 XML document. Furthermore, the program's exit code must be 292 expected_exit_code. 293 """ 294 295 actual = self._GetXmlOutput(gtest_prog_name, extra_args or [], 296 expected_exit_code) 297 expected = minidom.parseString(expected_xml) 298 self.NormalizeXml(actual.documentElement) 299 self.AssertEquivalentNodes(expected.documentElement, 300 actual.documentElement) 301 expected.unlink() 302 actual.unlink() 303 304 305 if __name__ == '__main__': 306 os.environ['GTEST_STACK_TRACE_DEPTH'] = '1' 307 gtest_test_utils.Main() 308
