1 #!/usr/bin/env python 2 # 3 # Copyright 2006, Google Inc. 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions are 8 # met: 9 # 10 # * Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # * Redistributions in binary form must reproduce the above 13 # copyright notice, this list of conditions and the following disclaimer 14 # in the documentation and/or other materials provided with the 15 # distribution. 16 # * Neither the name of Google Inc. nor the names of its 17 # contributors may be used to endorse or promote products derived from 18 # this software without specific prior written permission. 19 # 20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32 """Unit test for the gtest_xml_output module""" 33 34 __author__ = 'eefacm (at] gmail.com (Sean Mcafee)' 35 36 import datetime 37 import errno 38 import os 39 import re 40 import sys 41 from xml.dom import minidom, Node 42 43 import gtest_test_utils 44 import gtest_xml_test_utils 45 46 47 GTEST_LIST_TESTS_FLAG = '--gtest_list_tests' 48 GTEST_OUTPUT_FLAG = "--gtest_output" 49 GTEST_DEFAULT_OUTPUT_FILE = "test_detail.xml" 50 GTEST_PROGRAM_NAME = "gtest_xml_output_unittest_" 51 52 SUPPORTS_STACK_TRACES = False 53 54 if SUPPORTS_STACK_TRACES: 55 STACK_TRACE_TEMPLATE = '\nStack trace:\n*' 56 else: 57 STACK_TRACE_TEMPLATE = '' 58 59 EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?> 60 <testsuites tests="23" failures="4" disabled="2" errors="0" time="*" timestamp="*" name="AllTests"> 61 <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" errors="0" time="*"> 62 <testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/> 63 </testsuite> 64 <testsuite name="FailedTest" tests="1" failures="1" disabled="0" errors="0" time="*"> 65 <testcase name="Fails" status="run" time="*" classname="FailedTest"> 66 <failure message="gtest_xml_output_unittest_.cc:*
Value of: 2
Expected: 1" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 67 Value of: 2 68 Expected: 1%(stack)s]]></failure> 69 </testcase> 70 </testsuite> 71 <testsuite name="MixedResultTest" tests="3" failures="1" disabled="1" errors="0" time="*"> 72 <testcase name="Succeeds" status="run" time="*" classname="MixedResultTest"/> 73 <testcase name="Fails" status="run" time="*" classname="MixedResultTest"> 74 <failure message="gtest_xml_output_unittest_.cc:*
Value of: 2
Expected: 1" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 75 Value of: 2 76 Expected: 1%(stack)s]]></failure> 77 <failure message="gtest_xml_output_unittest_.cc:*
Value of: 3
Expected: 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 78 Value of: 3 79 Expected: 2%(stack)s]]></failure> 80 </testcase> 81 <testcase name="DISABLED_test" status="notrun" time="*" classname="MixedResultTest"/> 82 </testsuite> 83 <testsuite name="XmlQuotingTest" tests="1" failures="1" disabled="0" errors="0" time="*"> 84 <testcase name="OutputsCData" status="run" time="*" classname="XmlQuotingTest"> 85 <failure message="gtest_xml_output_unittest_.cc:*
Failed
XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]></top>" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 86 Failed 87 XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]>]]><![CDATA[</top>%(stack)s]]></failure> 88 </testcase> 89 </testsuite> 90 <testsuite name="InvalidCharactersTest" tests="1" failures="1" disabled="0" errors="0" time="*"> 91 <testcase name="InvalidCharactersInMessage" status="run" time="*" classname="InvalidCharactersTest"> 92 <failure message="gtest_xml_output_unittest_.cc:*
Failed
Invalid characters in brackets []" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 93 Failed 94 Invalid characters in brackets []%(stack)s]]></failure> 95 </testcase> 96 </testsuite> 97 <testsuite name="DisabledTest" tests="1" failures="0" disabled="1" errors="0" time="*"> 98 <testcase name="DISABLED_test_not_run" status="notrun" time="*" classname="DisabledTest"/> 99 </testsuite> 100 <testsuite name="PropertyRecordingTest" tests="4" failures="0" disabled="0" errors="0" time="*"> 101 <testcase name="OneProperty" status="run" time="*" classname="PropertyRecordingTest" key_1="1"/> 102 <testcase name="IntValuedProperty" status="run" time="*" classname="PropertyRecordingTest" key_int="1"/> 103 <testcase name="ThreeProperties" status="run" time="*" classname="PropertyRecordingTest" key_1="1" key_2="2" key_3="3"/> 104 <testcase name="TwoValuesForOneKeyUsesLastValue" status="run" time="*" classname="PropertyRecordingTest" key_1="2"/> 105 </testsuite> 106 <testsuite name="NoFixtureTest" tests="3" failures="0" disabled="0" errors="0" time="*"> 107 <testcase name="RecordProperty" status="run" time="*" classname="NoFixtureTest" key="1"/> 108 <testcase name="ExternalUtilityThatCallsRecordIntValuedProperty" status="run" time="*" classname="NoFixtureTest" key_for_utility_int="1"/> 109 <testcase name="ExternalUtilityThatCallsRecordStringValuedProperty" status="run" time="*" classname="NoFixtureTest" key_for_utility_string="1"/> 110 </testsuite> 111 <testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" errors="0" time="*"> 112 <testcase name="HasValueParamAttribute/0" value_param="33" status="run" time="*" classname="Single/ValueParamTest" /> 113 <testcase name="HasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" /> 114 <testcase name="AnotherTestThatHasValueParamAttribute/0" value_param="33" status="run" time="*" classname="Single/ValueParamTest" /> 115 <testcase name="AnotherTestThatHasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" /> 116 </testsuite> 117 <testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" errors="0" time="*"> 118 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="TypedTest/0" /> 119 </testsuite> 120 <testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" errors="0" time="*"> 121 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="TypedTest/1" /> 122 </testsuite> 123 <testsuite name="Single/TypeParameterizedTestCase/0" tests="1" failures="0" disabled="0" errors="0" time="*"> 124 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="Single/TypeParameterizedTestCase/0" /> 125 </testsuite> 126 <testsuite name="Single/TypeParameterizedTestCase/1" tests="1" failures="0" disabled="0" errors="0" time="*"> 127 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="Single/TypeParameterizedTestCase/1" /> 128 </testsuite> 129 </testsuites>""" % {'stack': STACK_TRACE_TEMPLATE} 130 131 132 EXPECTED_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?> 133 <testsuites tests="0" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests"> 134 </testsuites>""" 135 136 GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME) 137 138 SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess( 139 [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output 140 141 142 class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase): 143 """ 144 Unit test for Google Test's XML output functionality. 145 """ 146 147 # This test currently breaks on platforms that do not support typed and 148 # type-parameterized tests, so we don't run it under them. 149 if SUPPORTS_TYPED_TESTS: 150 def testNonEmptyXmlOutput(self): 151 """ 152 Runs a test program that generates a non-empty XML output, and 153 tests that the XML output is expected. 154 """ 155 self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY_XML, 1) 156 157 def testEmptyXmlOutput(self): 158 """Verifies XML output for a Google Test binary without actual tests. 159 160 Runs a test program that generates an empty XML output, and 161 tests that the XML output is expected. 162 """ 163 164 self._TestXmlOutput('gtest_no_test_unittest', EXPECTED_EMPTY_XML, 0) 165 166 def testTimestampValue(self): 167 """Checks whether the timestamp attribute in the XML output is valid. 168 169 Runs a test program that generates an empty XML output, and checks if 170 the timestamp attribute in the testsuites tag is valid. 171 """ 172 actual = self._GetXmlOutput('gtest_no_test_unittest', 0) 173 date_time_str = actual.documentElement.getAttributeNode('timestamp').value 174 # datetime.strptime() is only available in Python 2.5+ so we have to 175 # parse the expected datetime manually. 176 match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str) 177 self.assertTrue( 178 re.match, 179 'XML datettime string %s has incorrect format' % date_time_str) 180 date_time_from_xml = datetime.datetime( 181 year=int(match.group(1)), month=int(match.group(2)), 182 day=int(match.group(3)), hour=int(match.group(4)), 183 minute=int(match.group(5)), second=int(match.group(6))) 184 185 time_delta = abs(datetime.datetime.now() - date_time_from_xml) 186 # timestamp value should be near the current local time 187 self.assertTrue(time_delta < datetime.timedelta(seconds=600), 188 'time_delta is %s' % time_delta) 189 actual.unlink() 190 191 def testDefaultOutputFile(self): 192 """ 193 Confirms that Google Test produces an XML output file with the expected 194 default name if no name is explicitly specified. 195 """ 196 output_file = os.path.join(gtest_test_utils.GetTempDir(), 197 GTEST_DEFAULT_OUTPUT_FILE) 198 gtest_prog_path = gtest_test_utils.GetTestExecutablePath( 199 'gtest_no_test_unittest') 200 try: 201 os.remove(output_file) 202 except OSError, e: 203 if e.errno != errno.ENOENT: 204 raise 205 206 p = gtest_test_utils.Subprocess( 207 [gtest_prog_path, '%s=xml' % GTEST_OUTPUT_FLAG], 208 working_dir=gtest_test_utils.GetTempDir()) 209 self.assert_(p.exited) 210 self.assertEquals(0, p.exit_code) 211 self.assert_(os.path.isfile(output_file)) 212 213 def testSuppressedXmlOutput(self): 214 """ 215 Tests that no XML file is generated if the default XML listener is 216 shut down before RUN_ALL_TESTS is invoked. 217 """ 218 219 xml_path = os.path.join(gtest_test_utils.GetTempDir(), 220 GTEST_PROGRAM_NAME + 'out.xml') 221 if os.path.isfile(xml_path): 222 os.remove(xml_path) 223 224 command = [GTEST_PROGRAM_PATH, 225 '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path), 226 '--shut_down_xml'] 227 p = gtest_test_utils.Subprocess(command) 228 if p.terminated_by_signal: 229 # p.signal is avalable only if p.terminated_by_signal is True. 230 self.assertFalse( 231 p.terminated_by_signal, 232 '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal)) 233 else: 234 self.assert_(p.exited) 235 self.assertEquals(1, p.exit_code, 236 "'%s' exited with code %s, which doesn't match " 237 'the expected exit code %s.' 238 % (command, p.exit_code, 1)) 239 240 self.assert_(not os.path.isfile(xml_path)) 241 242 def _GetXmlOutput(self, gtest_prog_name, expected_exit_code): 243 """ 244 Returns the xml output generated by running the program gtest_prog_name. 245 Furthermore, the program's exit code must be expected_exit_code. 246 """ 247 xml_path = os.path.join(gtest_test_utils.GetTempDir(), 248 gtest_prog_name + 'out.xml') 249 gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name) 250 251 command = [gtest_prog_path, '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path)] 252 p = gtest_test_utils.Subprocess(command) 253 if p.terminated_by_signal: 254 self.assert_(False, 255 '%s was killed by signal %d' % (gtest_prog_name, p.signal)) 256 else: 257 self.assert_(p.exited) 258 self.assertEquals(expected_exit_code, p.exit_code, 259 "'%s' exited with code %s, which doesn't match " 260 'the expected exit code %s.' 261 % (command, p.exit_code, expected_exit_code)) 262 actual = minidom.parse(xml_path) 263 return actual 264 265 def _TestXmlOutput(self, gtest_prog_name, expected_xml, expected_exit_code): 266 """ 267 Asserts that the XML document generated by running the program 268 gtest_prog_name matches expected_xml, a string containing another 269 XML document. Furthermore, the program's exit code must be 270 expected_exit_code. 271 """ 272 273 actual = self._GetXmlOutput(gtest_prog_name, expected_exit_code) 274 expected = minidom.parseString(expected_xml) 275 self.NormalizeXml(actual.documentElement) 276 self.AssertEquivalentNodes(expected.documentElement, 277 actual.documentElement) 278 expected.unlink() 279 actual.unlink() 280 281 282 if __name__ == '__main__': 283 os.environ['GTEST_STACK_TRACE_DEPTH'] = '1' 284 gtest_test_utils.Main() 285
